From 4f8ddb824b346011982a9aa62d9698839a1d49cc Mon Sep 17 00:00:00 2001 From: Rovanion Luckey Date: Wed, 19 Apr 2023 14:20:56 +0200 Subject: [PATCH 001/430] Build reproducible tarballs from git --- easybuild/tools/filetools.py | 14 ++++--- test/framework/filetools.py | 72 +++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 80a6ba6560..7a431c8ee5 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2690,11 +2690,15 @@ def get_source_tarball_from_git(filename, targetdir, git_config): for cmd in cmds: run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name, trace=False) - # create an archive and delete the git repo directory - if keep_git_dir: - tar_cmd = ['tar', 'cfvz', targetpath, repo_name] - else: - tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] + # When CentOS 7 is phased out and tar>1.28 is everywhere, replace find-sort-pipe with tar-flag + # '--sort=name' and place LC_ALL in front of tar. Also remove flags --null, --no-recursion, and + # --files-from - from the flags to tar. See https://reproducible-builds.org/docs/archives/ + tar_cmd = ['find', repo_name, '-print0', '-path \'*/.git\' -prune' if not keep_git_dir else '', '|', + 'LC_ALL=C', 'sort', '--zero-terminated', '|', + 'GZIP=--no-name', 'tar', '--create', '--file', targetpath, '--no-recursion', + '--gzip', '--mtime="1970-01-01 00:00Z"', '--owner=0', '--group=0', + '--numeric-owner', '--format=gnu', '--null', + '--no-recursion', '--files-from -'] run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False, trace=False) # cleanup (repo_name dir does not exist in dry run mode) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index fcaebe16d4..5e2d0412fe 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2787,41 +2787,57 @@ def run_check(): 'url': 'git@github.com:easybuilders', 'tag': 'tag_for_tests', } - git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter + string_args = { + 'git_repo': 'git@github.com:easybuilders/testrepository.git', + 'test_prefix': self.test_prefix, + } + expected = '\n'.join([ r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', - r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - r" \(in /.*\)", - ]) % git_repo + r" \(in .*/tmp.*\)", + r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file %(test_prefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() git_config['clone_into'] = 'test123' expected = '\n'.join([ r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', - r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', - r" \(in /.*\)", - ]) % git_repo + r" \(in .*/tmp.*\)", + r' running command "find test123 -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() del git_config['clone_into'] git_config['recursive'] = True expected = '\n'.join([ r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', - r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - r" \(in /.*\)", - ]) % git_repo + r" \(in .*/tmp.*\)", + r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() git_config['keep_git_dir'] = True expected = '\n'.join([ r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', - r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz testrepository"', - r" \(in /.*\)", - ]) % git_repo + r" \(in .*/tmp.*\)", + r' running command "find testrepository -print0 | LC_ALL=C sort --zero-terminated | GZIP=--no-name tar' + r' --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion --gzip' + r' --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu --null --no-recursion' + r' --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() del git_config['keep_git_dir'] @@ -2829,23 +2845,29 @@ def run_check(): git_config['commit'] = '8456f86' expected = '\n'.join([ r' running command "git clone --no-checkout %(git_repo)s"', - r" \(in /.*\)", + r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - r" \(in /.*\)", - ]) % git_repo + r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() del git_config['recursive'] expected = '\n'.join([ r' running command "git clone --no-checkout %(git_repo)s"', - r" \(in /.*\)", + r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86"', r" \(in testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - r" \(in /.*\)", - ]) % git_repo + r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() # Test with real data. From a3a427bf76b06c6ab6ff88fcc8a500d2eda38449 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 09:54:36 +0000 Subject: [PATCH 002/430] detect Fortran .mod files in GCCcore installations --- easybuild/framework/easyblock.py | 21 +++++++++++++++++++++ easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + 3 files changed, 23 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 52a700bdfe..7d31953520 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3303,6 +3303,19 @@ def regex_for_lib(lib): return fail_msg + def sanity_check_mod_files(self): + """ + Check installation for Fortran .mod files + """ + self.log.debug("Check for .mod files in install directory") + mod_files = glob.glob(os.path.join(self.installdir, '**', '*.mod'), recursive=True) + + fail_msg = None + if mod_files: + fail_msg = ".mod files (%s) found in the installation." % ', '.join(mod_files) + + return fail_msg + def _sanity_check_step_common(self, custom_paths, custom_commands): """ Determine sanity check paths and commands to use. @@ -3625,6 +3638,14 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) + if self.toolchain.name in ['GCCcore']: + mod_files_found = self.sanity_check_mod_files() + if mod_files_found: + if build_option('fail_on_mod_files'): + self.sanity_check_fail_msgs.append(mod_files_found) + else: + print_warning(mod_files_found) + # cleanup if self.fake_mod_data: self.clean_up_fake_module(self.fake_mod_data) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b18bae480c..c5db28b05f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -277,6 +277,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'enforce_checksums', 'experimental', 'extended_dry_run', + 'fail_on_mod_files', 'force', 'generate_devel_module', 'group_writable_installdir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 999b62bb5c..faa1f48e80 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,6 +400,7 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), + 'fail-on-mod-files': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " From f148ffbf6e67fa49e13cc9f317d21e7576678264 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 10:19:40 +0000 Subject: [PATCH 003/430] easyconfig param to skip check --- easybuild/framework/easyconfig/default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dd91229d1e..74b96cffde 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -130,6 +130,7 @@ 'sanity_check_paths': [{}, ("List of files and directories to check " "(format: {'files':, 'dirs':})"), BUILD], 'skip': [False, "Skip existing software", BUILD], + 'skip_mod_files_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], 'skipsteps': [[], "Skip these steps", BUILD], 'source_urls': [[], "List of URLs for source files", BUILD], 'sources': [[], "List of source files", BUILD], From ec6ece644c7b87d43515c3317d1e0e1688088e60 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 10:46:55 +0000 Subject: [PATCH 004/430] skip check when not needed --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7d31953520..f789ffea2b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3638,7 +3638,7 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore']: + if self.toolchain.name in ['GCCcore'] and not self.cfg._config['skip_mod_files_check']: mod_files_found = self.sanity_check_mod_files() if mod_files_found: if build_option('fail_on_mod_files'): From d887426f83df97dbd9c5d26bd78747cba7efb0eb Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 11:00:12 +0000 Subject: [PATCH 005/430] correctly check for if the parameter is set --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f789ffea2b..a57db2591a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3638,7 +3638,7 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore'] and not self.cfg._config['skip_mod_files_check']: + if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_check']: mod_files_found = self.sanity_check_mod_files() if mod_files_found: if build_option('fail_on_mod_files'): From 809b6948d5f024fe0e9e137eb5c972834a09dcf6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 12:52:53 +0000 Subject: [PATCH 006/430] add testing --- test/framework/toy_build.py | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 40384d7582..54582ec48e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3955,6 +3955,44 @@ def test_toy_build_sanity_check_linked_libs(self): self._test_toy_build(ec_file=test_ec, extra_args=args, force=False, raise_error=True, verbose=False, verify=False) + def test_toy_mod_files(self): + """Check detection of .mod files""" + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + test_ec_txt = read_file(toy_ec) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec) + + test_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/lib/file.mod']" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec) + + args = ['--try-toolchain=GCCcore,6.2.0', '--disable-map-toolchains'] + self.mock_stdout(True) + self.mock_stderr(True) + self._test_toy_build(ec_file=test_ec, extra_args=args) + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + pattern = r"WARNING: .mod files (.*) found in the installation." + self.assertRegex(stderr.strip(), pattern) + + args += ['--fail-on-mod-files'] + pattern = r"Sanity check failed: .mod files (.*) found in the installation." + self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, + extra_args=args, verify=False, fails=True, verbose=False, raise_error=True) + + test_ec_txt += "\nskip_mod_files_check = True" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec, extra_args=args) + def test_toy_ignore_test_failure(self): """Check whether use of --ignore-test-failure is mentioned in build output.""" args = ['--ignore-test-failure'] From 0b3daf25fcb1237c39d867028d9045ebeff82aab Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 13:03:42 +0000 Subject: [PATCH 007/430] address review comments --- easybuild/framework/easyblock.py | 14 +++++++------- easybuild/framework/easyconfig/default.py | 2 +- easybuild/tools/config.py | 2 +- easybuild/tools/options.py | 2 +- test/framework/toy_build.py | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a57db2591a..954bb3212e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3307,7 +3307,7 @@ def sanity_check_mod_files(self): """ Check installation for Fortran .mod files """ - self.log.debug("Check for .mod files in install directory") + self.log.debug(f"Checking for .mod files in install directory {self.installdir}...") mod_files = glob.glob(os.path.join(self.installdir, '**', '*.mod'), recursive=True) fail_msg = None @@ -3638,13 +3638,13 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_check']: - mod_files_found = self.sanity_check_mod_files() - if mod_files_found: - if build_option('fail_on_mod_files'): - self.sanity_check_fail_msgs.append(mod_files_found) + if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_sanity_check']: + mod_files_found_msg = self.sanity_check_mod_files() + if mod_files_found_msg: + if build_option('fail_on_mod_files_gcccore'): + self.sanity_check_fail_msgs.append(mod_files_found_msg) else: - print_warning(mod_files_found) + print_warning(mod_files_found_msg) # cleanup if self.fake_mod_data: diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 74b96cffde..5de9799e54 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -130,7 +130,7 @@ 'sanity_check_paths': [{}, ("List of files and directories to check " "(format: {'files':, 'dirs':})"), BUILD], 'skip': [False, "Skip existing software", BUILD], - 'skip_mod_files_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], + 'skip_mod_files_sanity_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], 'skipsteps': [[], "Skip these steps", BUILD], 'source_urls': [[], "List of URLs for source files", BUILD], 'sources': [[], "List of source files", BUILD], diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index c5db28b05f..d8e37ac0da 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -277,7 +277,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'enforce_checksums', 'experimental', 'extended_dry_run', - 'fail_on_mod_files', + 'fail_on_mod_files_gcccore', 'force', 'generate_devel_module', 'group_writable_installdir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index faa1f48e80..766b891bda 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,7 +400,7 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), - 'fail-on-mod-files': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), + 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 54582ec48e..9bf85607fa 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3982,12 +3982,12 @@ def test_toy_mod_files(self): pattern = r"WARNING: .mod files (.*) found in the installation." self.assertRegex(stderr.strip(), pattern) - args += ['--fail-on-mod-files'] + args += ['--fail-on-mod-files-gcccore'] pattern = r"Sanity check failed: .mod files (.*) found in the installation." self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, extra_args=args, verify=False, fails=True, verbose=False, raise_error=True) - test_ec_txt += "\nskip_mod_files_check = True" + test_ec_txt += "\nskip_mod_files_sanity_check = True" write_file(test_ec, test_ec_txt) with self.mocked_stdout_stderr(): From 1644c930c5126edd73311395d313d4b0ddb4b109 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 13:04:48 +0000 Subject: [PATCH 008/430] appease the hound --- easybuild/tools/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 766b891bda..1d0330ee30 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,7 +400,8 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), - 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), + 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', + False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " From 0700f2dc7fdcf66e39533cf1478c05540da982ea Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 09:53:44 +0100 Subject: [PATCH 009/430] rename Extension.run() method to install_extension() --- easybuild/framework/easyblock.py | 4 ++-- easybuild/framework/extension.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 4 ++-- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 52a700bdfe..622e806094 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1914,12 +1914,12 @@ def install_extensions_sequential(self, install=True): ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, rpath_filter_dirs=self.rpath_filter_dirs) - # real work + # actual installation of the extension if install: try: ext.prerun() with self.module_generator.start_module_creation(): - txt = ext.run() + txt = ext.install_extension() if txt: self.module_extra_extensions += txt ext.postrun() diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 8dc4669392..dac48636ef 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -177,7 +177,7 @@ def prerun(self): """ pass - def run(self, *args, **kwargs): + def install_extension(self, *args, **kwargs): """ Actual installation of an extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 9c700cf779..42f56d6478 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -59,7 +59,7 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) - def run(self, *args, **kwargs): + def install_extension(self, *args, **kwargs): """ Install toy extension. """ @@ -78,7 +78,7 @@ def prerun(self): super(Toy_Extension, self).prerun() if self.src: - super(Toy_Extension, self).run(unpack_src=True) + super(Toy_Extension, self).install_extension(unpack_src=True) EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) def run_async(self): diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index c4614e3333..b361df126b 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -154,10 +154,10 @@ def prerun(self): """ Prepare installation of toy as extension. """ - super(EB_toy, self).run(unpack_src=True) + super(EB_toy, self).install_extension(unpack_src=True) self.configure_step() - def run(self): + def install_extension(self): """ Install toy as extension. """ From 202c78ed725dea5fd389a693ccb93866728e1569 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 15:30:54 +0100 Subject: [PATCH 010/430] rename ExtensionEasyblock.run() to install_extension()" --- easybuild/framework/extensioneasyblock.py | 2 +- test/framework/easyblock.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 6ad116b20b..c06fb56e0f 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -133,7 +133,7 @@ def _set_start_dir(self): self.log.warning(warn_msg) print_warning(warn_msg, silent=build_option('silent')) - def run(self, unpack_src=False): + def install_extension(self, unpack_src=False): """Common operations for extensions: unpacking sources, patching, ...""" # unpack file if desired diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 9f656573d1..ae1ebcd56d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2207,7 +2207,7 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): eb.extensions_step(fetch=True, install=False) # extract sources of the extension ext = eb.ext_instances[-1] - ext.run(unpack_src=unpack_src) + ext.install_extension(unpack_src=unpack_src) if expected_start_dir is None: self.assertIsNone(ext.start_dir) From d9cb50e8618421a67aa5ea3a4ac31a0747cd7c20 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 09:56:05 +0100 Subject: [PATCH 011/430] rename Extension.prerun() method to preinstall_extension() --- easybuild/framework/easyblock.py | 4 ++-- easybuild/framework/extension.py | 2 +- .../easyblocks/generic/toy_extension.py | 20 +++++++++---------- .../sandbox/easybuild/easyblocks/t/toy.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 622e806094..85cccc6baf 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1917,7 +1917,7 @@ def install_extensions_sequential(self, install=True): # actual installation of the extension if install: try: - ext.prerun() + ext.preinstall_extension() with self.module_generator.start_module_creation(): txt = ext.install_extension() if txt: @@ -2054,7 +2054,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, rpath_filter_dirs=self.rpath_filter_dirs) if install: - ext.prerun() + ext.preinstall_extension() ext.run_async() running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index dac48636ef..2330ab348f 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -171,7 +171,7 @@ def version(self): """ return self.ext.get('version', None) - def prerun(self): + def preinstall_extension(self): """ Stuff to do before installing a extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 42f56d6478..2a15a2cb3b 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -59,6 +59,16 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) + def preinstall_extension(self): + """ + Prepare installation of toy extension. + """ + super(Toy_Extension, self).preinstall_extension() + + if self.src: + super(Toy_Extension, self).install_extension(unpack_src=True) + EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) + def install_extension(self, *args, **kwargs): """ Install toy extension. @@ -71,16 +81,6 @@ def install_extension(self, *args, **kwargs): return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name) - def prerun(self): - """ - Prepare installation of toy extension. - """ - super(Toy_Extension, self).prerun() - - if self.src: - super(Toy_Extension, self).install_extension(unpack_src=True) - EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) - def run_async(self): """ Install toy extension asynchronously. diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index b361df126b..0a6ca8e7a8 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -150,7 +150,7 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) - def prerun(self): + def preinstall_extension(self): """ Prepare installation of toy as extension. """ From f85564b9c7ba6dfcc21784a9d24edaba1d15e126 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 09:57:54 +0100 Subject: [PATCH 012/430] rename Extension.postrun() method to postinstall_extension() --- easybuild/framework/easyblock.py | 4 ++-- easybuild/framework/extension.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 4 ++-- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 85cccc6baf..5d9c37cec4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1922,7 +1922,7 @@ def install_extensions_sequential(self, install=True): txt = ext.install_extension() if txt: self.module_extra_extensions += txt - ext.postrun() + ext.postinstall_extension() finally: if not self.dry_run: ext_duration = datetime.now() - start_time @@ -1983,7 +1983,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): for ext in running_exts[:]: if self.dry_run or ext.async_cmd_check(): self.log.info("Installation of %s completed!", ext.name) - ext.postrun() + ext.postinstall_extension() running_exts.remove(ext) installed_ext_names.append(ext.name) update_exts_progress_bar_helper(running_exts, 1) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 2330ab348f..b84ddf8d8b 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -189,7 +189,7 @@ def run_async(self, *args, **kwargs): """ raise NotImplementedError - def postrun(self): + def postinstall_extension(self): """ Stuff to do after installing a extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 2a15a2cb3b..07139c92ec 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -91,11 +91,11 @@ def run_async(self): else: self.async_cmd_info = False - def postrun(self): + def postinstall_extension(self): """ Wrap up installation of toy extension. """ - super(Toy_Extension, self).postrun() + super(Toy_Extension, self).postinstall_extension() EB_toy.install_step(self.master, name=self.name) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 0a6ca8e7a8..e84a40c1c7 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -170,7 +170,7 @@ def run_async(self): cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) self.async_cmd_start(cmd) - def postrun(self): + def postinstall_extension(self): """ Wrap up installation of toy as extension. """ From 23446dfb4521be967f2f797775f7396be102e042 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 09:59:14 +0100 Subject: [PATCH 013/430] rename Extension.run_async() method to install_extension_async() --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/extension.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5d9c37cec4..a6c111c29e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2055,7 +2055,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): rpath_filter_dirs=self.rpath_filter_dirs) if install: ext.preinstall_extension() - ext.run_async() + ext.install_extension_async() running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar_helper(running_exts, 0) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index b84ddf8d8b..a9fff98020 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -183,7 +183,7 @@ def install_extension(self, *args, **kwargs): """ pass - def run_async(self, *args, **kwargs): + def install_extension_async(self, *args, **kwargs): """ Asynchronous installation of an extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 07139c92ec..204da5458f 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -81,7 +81,7 @@ def install_extension(self, *args, **kwargs): return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name) - def run_async(self): + def install_extension_async(self): """ Install toy extension asynchronously. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index e84a40c1c7..f87696f122 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -163,7 +163,7 @@ def install_extension(self): """ self.build_step() - def run_async(self): + def install_extension_async(self): """ Asynchronous installation of toy as extension. """ From 845d11085d83f128e58c92789331956d039da5d4 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 15:25:37 +0100 Subject: [PATCH 014/430] rename EasyBlock.install_extensions() to install_all_extensions() --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a6c111c29e..62cbe9873d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1850,7 +1850,7 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances - def install_extensions(self, install=True): + def install_all_extensions(self, install=True): """ Install extensions. @@ -2883,7 +2883,7 @@ def extensions_step(self, fetch=False, install=True): if self.skip: self.skip_extensions() - self.install_extensions(install=install) + self.install_all_extensions(install=install) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: From 6b00163e2ed6d674881f8b6261de0fff3cdf633c Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 13 Dec 2023 16:41:04 +0100 Subject: [PATCH 015/430] separate prefix in preinstall_extension and postinstall_extension method names --- easybuild/framework/easyblock.py | 8 ++++---- easybuild/framework/extension.py | 4 ++-- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 8 ++++---- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 62cbe9873d..fe45be97eb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1917,12 +1917,12 @@ def install_extensions_sequential(self, install=True): # actual installation of the extension if install: try: - ext.preinstall_extension() + ext.pre_install_extension() with self.module_generator.start_module_creation(): txt = ext.install_extension() if txt: self.module_extra_extensions += txt - ext.postinstall_extension() + ext.post_install_extension() finally: if not self.dry_run: ext_duration = datetime.now() - start_time @@ -1983,7 +1983,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): for ext in running_exts[:]: if self.dry_run or ext.async_cmd_check(): self.log.info("Installation of %s completed!", ext.name) - ext.postinstall_extension() + ext.post_install_extension() running_exts.remove(ext) installed_ext_names.append(ext.name) update_exts_progress_bar_helper(running_exts, 1) @@ -2054,7 +2054,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, rpath_filter_dirs=self.rpath_filter_dirs) if install: - ext.preinstall_extension() + ext.pre_install_extension() ext.install_extension_async() running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index a9fff98020..e2cbe2c323 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -171,7 +171,7 @@ def version(self): """ return self.ext.get('version', None) - def preinstall_extension(self): + def pre_install_extension(self): """ Stuff to do before installing a extension. """ @@ -189,7 +189,7 @@ def install_extension_async(self, *args, **kwargs): """ raise NotImplementedError - def postinstall_extension(self): + def post_install_extension(self): """ Stuff to do after installing a extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 204da5458f..34aa1eb259 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -59,11 +59,11 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) - def preinstall_extension(self): + def pre_install_extension(self): """ Prepare installation of toy extension. """ - super(Toy_Extension, self).preinstall_extension() + super(Toy_Extension, self).pre_install_extension() if self.src: super(Toy_Extension, self).install_extension(unpack_src=True) @@ -91,11 +91,11 @@ def install_extension_async(self): else: self.async_cmd_info = False - def postinstall_extension(self): + def post_install_extension(self): """ Wrap up installation of toy extension. """ - super(Toy_Extension, self).postinstall_extension() + super(Toy_Extension, self).post_install_extension() EB_toy.install_step(self.master, name=self.name) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index f87696f122..3ff4b73d67 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -150,7 +150,7 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) - def preinstall_extension(self): + def pre_install_extension(self): """ Prepare installation of toy as extension. """ @@ -170,7 +170,7 @@ def install_extension_async(self): cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) self.async_cmd_start(cmd) - def postinstall_extension(self): + def post_install_extension(self): """ Wrap up installation of toy as extension. """ From 9d92bcea5f43eac5e0128ea3841a9fb2c3444533 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 13 Dec 2023 16:57:35 +0100 Subject: [PATCH 016/430] add deprecated extension install methods with deprecation warnings --- easybuild/framework/easyblock.py | 8 +++++++ easybuild/framework/extension.py | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index fe45be97eb..914e8ad4a0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1850,6 +1850,14 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances + def install_extensions(self, *args, **kwargs): + """[DEPRECATED] Install extensions.""" + self.log.deprecated( + "Easyblock.install_extensions() is deprecated, use Easyblock.install_all_extensions() instead.", + '5.0', + ) + self.install_all_extensions(*args, **kwargs) + def install_all_extensions(self, install=True): """ Install extensions. diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index e2cbe2c323..31cfb8d373 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -171,24 +171,64 @@ def version(self): """ return self.ext.get('version', None) + def prerun(self): + """ + [DEPRECATED] Stuff to do before installing a extension. + """ + self.log.deprecated( + "Extension.prerun() is deprecated, use Extension.pre_install_extension() instead.", + '5.0', + ) + self.pre_install_extension() + def pre_install_extension(self): """ Stuff to do before installing a extension. """ pass + def run(self, *args, **kwargs): + """ + [DEPRECATED] Actual installation of an extension. + """ + self.log.deprecated( + "Extension.run() is deprecated, use Extension.install_extension() instead.", + '5.0', + ) + self.install_extension(*args, **kwargs) + def install_extension(self, *args, **kwargs): """ Actual installation of an extension. """ pass + def run_async(self, *args, **kwargs): + """ + [DEPRECATED] Asynchronous installation of an extension. + """ + self.log.deprecated( + "Extension.run_async() is deprecated, use Extension.install_extension_async() instead.", + '5.0', + ) + self.install_extension_async(*args, **kwargs) + def install_extension_async(self, *args, **kwargs): """ Asynchronous installation of an extension. """ raise NotImplementedError + def postrun(self): + """ + [DEPRECATED] Stuff to do after installing a extension. + """ + self.log.deprecated( + "Extension.postrun() is deprecated, use Extension.post_install_extension() instead.", + '5.0', + ) + self.post_install_extension() + def post_install_extension(self): """ Stuff to do after installing a extension. From e50ffb940c95a412fa512c9a0c18f246511db4d0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 13:40:33 +0100 Subject: [PATCH 017/430] make `is_rpath_wrapper` faster Avoid uneccessary read of the FULL binary file if it cannot be an RPATH wrapper because it is in the wrong folder. --- easybuild/tools/toolchain/toolchain.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index a16fe35dc0..17bc9e881f 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -941,10 +941,11 @@ def is_rpath_wrapper(path): """ Check whether command at specified location already is an RPATH wrapper script rather than the actual command """ - in_rpath_wrappers_dir = os.path.basename(os.path.dirname(os.path.dirname(path))) == RPATH_WRAPPERS_SUBDIR + if os.path.basename(os.path.dirname(os.path.dirname(path))) != RPATH_WRAPPERS_SUBDIR: + return False + # Check if `rpath_args`` is called in the file # need to use binary mode to read the file, since it may be an actual compiler command (which is a binary file) - calls_rpath_args = b'rpath_args.py $CMD' in read_file(path, mode='rb') - return in_rpath_wrappers_dir and calls_rpath_args + return b'rpath_args.py $CMD' in read_file(path, mode='rb') def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None): """ From e346d636ac448b71825c0d5a5350ac11bc7f0595 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 19 Dec 2023 09:36:11 +0100 Subject: [PATCH 018/430] add terse support to `--missing-modules` It is useful to have the missing modules in a format that can be processed in a script, i.e. one EC name per line. So allow `--terse` in combination with `--missing-modules`. --- easybuild/main.py | 6 +++--- easybuild/tools/robot.py | 8 +++++--- test/framework/options.py | 24 ++++++++++++++++++------ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 73550e3998..826880aad1 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -495,7 +495,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: if options.missing_modules: - txt = missing_deps(easyconfigs, modtool) + txt = missing_deps(easyconfigs, modtool, terse=options.terse) else: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) @@ -684,7 +684,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr options.list_prs, options.merge_pr, options.review_pr, - options.terse, + options.terse and not options.missing_modules, search_query, ] if any(early_stop_options): @@ -721,7 +721,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if do_cleanup: - cleanup(logfile, eb_tmpdir, testing, silent=False) + cleanup(logfile, eb_tmpdir, testing, silent=options.terse) def prepare_main(args=None, logfile=None, testing=None): diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 68a4c9028b..3a21bc2f7f 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -288,16 +288,18 @@ def dry_run(easyconfigs, modtool, short=False): return '\n'.join(lines) -def missing_deps(easyconfigs, modtool): +def missing_deps(easyconfigs, modtool, terse=False): """ Determine subset of easyconfigs for which no module is installed yet. """ ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False) missing = skip_available(ordered_ecs, modtool) - if missing: + if terse: + lines = [os.path.basename(x['ec'].path) for x in missing] + elif missing: lines = ['', "%d out of %d required modules missing:" % (len(missing), len(ordered_ecs)), ''] - for ec in [x['ec'] for x in missing]: + for ec in (x['ec'] for x in missing): if ec.short_mod_name != ec.full_mod_name: modname = '%s | %s' % (ec.mod_subdir, ec.short_mod_name) else: diff --git a/test/framework/options.py b/test/framework/options.py index e6be49ab4f..a246311727 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1472,14 +1472,26 @@ def test_missing(self): ]) for opt in ['-M', '--missing-modules']: - self.mock_stderr(True) - self.mock_stdout(True) - self.eb_main(args + [opt], testing=False, raise_error=True) - stderr, stdout = self.get_stderr(), self.get_stdout() - self.mock_stderr(False) - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + self.eb_main(args + [opt], testing=False, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() self.assertFalse(stderr) self.assertIn(expected, stdout) + # --terse + with self.mocked_stdout_stderr(): + self.eb_main(args + ['-M', '--terse'], testing=False, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.assertFalse(stderr) + if mns == 'HierarchicalMNS': + expected = '\n'.join([ + "GCC-4.6.3.eb", + "intel-2018a.eb", + "toy-0.0-deps.eb", + "gzip-1.4-GCC-4.6.3.eb", + ]) + else: + expected = 'gzip-1.4-GCC-4.6.3.eb' + self.assertEqual(stdout, expected + '\n') def test_dry_run_short(self): """Test dry run (short format).""" From b1efd5ced3f46b2dad972ff3e0100d4962f454b5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 19 Dec 2023 10:06:59 +0100 Subject: [PATCH 019/430] Don't print index status when using `--terse` --- easybuild/tools/config.py | 1 + easybuild/tools/filetools.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6c0a173fe4..f025ce831c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -306,6 +306,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'skip_test_cases', 'skip_test_step', 'sticky_bit', + 'terse', 'trace', 'unit_testing_mode', 'upload_test_report', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6e4bd2d83e..21d9d5e5e0 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -966,11 +966,12 @@ def load_index(path, ignore_dirs=None): # check whether index is still valid if valid_ts: curr_ts = datetime.datetime.now() + terse = build_option('terse') if curr_ts > valid_ts: - print_warning("Index for %s is no longer valid (too old), so ignoring it...", path) + print_warning("Index for %s is no longer valid (too old), so ignoring it...", path, silent=terse) index = None else: - print_msg("found valid index for %s, so using it...", path) + print_msg("found valid index for %s, so using it...", path, silent=terse) return index or None From b1c5d082e1940b81da947203779937edb10bb993 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 10:22:58 +0100 Subject: [PATCH 020/430] Remove superflous string formatting `"%s" % var` is the same as `str(var)` while the latter is faster due to less parsing, inserting etc. In some places `var` was already a string so it can just be used. --- easybuild/base/generaloption.py | 8 ++++---- easybuild/framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/tools/docs.py | 6 +++--- easybuild/tools/filetools.py | 2 +- easybuild/tools/module_generator.py | 2 +- easybuild/tools/testing.py | 2 +- test/framework/module_generator.py | 8 ++++---- test/framework/options.py | 2 +- .../module_naming_scheme/test_module_naming_scheme.py | 4 ++-- test/framework/toy_build.py | 4 ++-- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 51f7daafe3..6a4c5f0d5d 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -90,7 +90,7 @@ def set_columns(cols=None): pass if cols is not None: - os.environ['COLUMNS'] = "%s" % cols + os.environ['COLUMNS'] = str(cols) def what_str_list_tuple(name): @@ -822,8 +822,8 @@ def get_env_options(self): self.environment_arguments.append("%s=%s" % (lo, val)) else: # interpretation of values: 0/no/false means: don't set it - if ("%s" % val).lower() not in ("0", "no", "false",): - self.environment_arguments.append("%s" % lo) + if str(val).lower() not in ("0", "no", "false",): + self.environment_arguments.append(str(lo)) else: self.log.debug("Environment variable %s is not set" % env_opt_name) @@ -1189,7 +1189,7 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non for extra_detail in details[4:]: if isinstance(extra_detail, (list, tuple,)): # choices - nameds['choices'] = ["%s" % x for x in extra_detail] # force to strings + nameds['choices'] = [str(x) for x in extra_detail] # force to strings hlp += ' (choices: %s)' % ', '.join(nameds['choices']) elif isinstance(extra_detail, string_type) and len(extra_detail) == 1: args.insert(0, "-%s" % extra_detail) diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 8abb697ece..9a1626c60b 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -370,7 +370,7 @@ def parse(self, configobj): for key, value in self.supported.items(): if key not in self.VERSION_OPERATOR_VALUE_TYPES: raise EasyBuildError('Unsupported key %s in %s section', key, self.SECTION_MARKER_SUPPORTED) - self.sections['%s' % key] = value + self.sections[key] = value for key, supported_key, fn_name in [('version', 'versions', 'get_version_str'), ('toolchain', 'toolchains', 'as_dict')]: diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index c3e26ab868..7aaa69b69a 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -324,7 +324,7 @@ def __repr__(self): newval = "%s + %s" % (fval, res.group('val')) _log.debug("Prepending %s to %s" % (fval, key)) else: - newval = "%s" % fval + newval = str(fval) _log.debug("Overwriting %s with %s" % (key, fval)) ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..4fcf1ad466 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -289,7 +289,7 @@ def avail_easyconfig_licenses_md(): lics = sorted(EASYCONFIG_LICENSES_DICT.items()) table_values = [ ["``%s``" % lic().name for _, lic in lics], - ["%s" % lic().description for _, lic in lics], + [lic().description for _, lic in lics], ["``%s``" % lic().version for _, lic in lics], ] @@ -1199,7 +1199,7 @@ def avail_toolchain_opts_md(name, tc_dict): tc_items = sorted(tc_dict.items()) table_values = [ ['``%s``' % val[0] for val in tc_items], - ['%s' % val[1][1] for val in tc_items], + [val[1][1] for val in tc_items], ['``%s``' % val[1][0] for val in tc_items], ] @@ -1217,7 +1217,7 @@ def avail_toolchain_opts_rst(name, tc_dict): tc_items = sorted(tc_dict.items()) table_values = [ ['``%s``' % val[0] for val in tc_items], - ['%s' % val[1][1] for val in tc_items], + [val[1][1] for val in tc_items], ['``%s``' % val[1][0] for val in tc_items], ] diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6e4bd2d83e..5a2f724bc8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2668,7 +2668,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd.append('%s/%s.git' % (url, repo_name)) if clone_into: - clone_cmd.append('%s' % clone_into) + clone_cmd.append(clone_into) tmpdir = tempfile.mkdtemp() cwd = change_dir(tmpdir) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 0c19b93c8b..45582bbe5c 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1289,7 +1289,7 @@ def get_description(self, conflict=True): extensions_list = self._generate_extensions_list() if extensions_list: - extensions_stmt = 'extensions("%s")' % ','.join(['%s' % x for x in extensions_list]) + extensions_stmt = 'extensions("%s")' % ','.join([str(x) for x in extensions_list]) # put this behind a Lmod version check as 'extensions' is only (well) supported since Lmod 8.2.8, # see https://lmod.readthedocs.io/en/latest/330_extensions.html#module-extensions and # https://github.com/TACC/Lmod/issues/428 diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 68eedc23ce..ffd8ce580b 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -178,7 +178,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ ]) test_report.extend([ "#### Test result", - "%s" % msg, + msg, "", ]) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index a0c6131140..d2fbb6f642 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -82,7 +82,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -107,7 +107,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -137,7 +137,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -161,7 +161,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", diff --git a/test/framework/options.py b/test/framework/options.py index f3173dbb87..3bfc863e41 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2190,7 +2190,7 @@ def test_ignore_osdeps(self): self.assertTrue(regex.search(outtxt), "OS dependencies are checked, outtxt: %s" % outtxt) msg = "One or more OS dependencies were not found: " msg += r"\[\('nosuchosdependency',\), \('nosuchdep_option1', 'nosuchdep_option2'\)\]" - regex = re.compile(r'%s' % msg, re.M) + regex = re.compile(msg, re.M) self.assertTrue(regex.search(outtxt), "OS dependencies are honored, outtxt: %s" % outtxt) # check whether OS dependencies are effectively ignored diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 5eb5f66682..02934e5745 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -64,6 +64,6 @@ def det_module_symlink_paths(self, ec): def is_short_modname_for(self, modname, name): """ - Determine whether the specified (short) module name is a module for software with the specified name. + Determine whether the specified (short) module name is a moduleq for software with the specified name. """ - return modname.find('%s' % name) != -1 + return modname.find(name) != -1 diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9ccb3c08df..cc84ce86d1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1491,7 +1491,7 @@ def test_toy_module_fulltxt(self): mod_txt_regex_pattern = '\n'.join([ r'help\(\[==\[', r'', - r'%s' % help_txt, + help_txt, r'\]==\]\)', r'', r'whatis\(\[==\[Description: Toy C program, 100% toy.\]==\]\)', @@ -1528,7 +1528,7 @@ def test_toy_module_fulltxt(self): r'proc ModulesHelp { } {', r' puts stderr {', r'', - r'%s' % help_txt, + help_txt, r' }', r'}', r'', From 116e6db20a1dec9014fea562b157e97cfed0b08e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 09:28:03 +0100 Subject: [PATCH 021/430] Cleanup & Speedup environment checks `os.environ` is a dict hence the check for existance and access can usually be combined into a single access of the key avoiding the repeated lookup. Especially for `os.getenv` which already returns `None` when the key doesn't exist. Also use `.pop(key, None)` to remove a key without checking for it's existance first to avoid C&P errors --- easybuild/base/fancylogger.py | 15 +++++++-------- easybuild/main.py | 3 +-- easybuild/tools/environment.py | 4 ++-- easybuild/tools/modules.py | 12 ++++-------- test/framework/environment.py | 3 +-- test/framework/filetools.py | 6 ++---- test/framework/module_generator.py | 3 +-- test/framework/modules.py | 12 ++++-------- test/framework/modulestool.py | 3 +-- test/framework/options.py | 12 ++++-------- test/framework/robot.py | 3 +-- test/framework/toolchain.py | 12 ++++-------- test/framework/toy_build.py | 3 +-- test/framework/utilities.py | 6 ++---- 14 files changed, 35 insertions(+), 62 deletions(-) diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index b5a63477c6..8b5f0a8178 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -147,10 +147,10 @@ def _env_to_boolean(varname, default=False): >>> _env_to_boolean('NO_FOOBAR') False """ - if varname not in os.environ: + try: + return os.environ[varname].lower() in ('1', 'yes', 'true', 'y') + except KeyError: return default - else: - return os.environ.get(varname).lower() in ('1', 'yes', 'true', 'y') OPTIMIZED_ANSWER = "not available in optimized mode" @@ -911,16 +911,15 @@ def resetroot(): _default_logTo = None if 'FANCYLOG_SERVER' in os.environ: server = os.environ['FANCYLOG_SERVER'] - port = DEFAULT_UDP_PORT if ':' in server: server, port = server.split(':') + else: + port = DEFAULT_UDP_PORT # maybe the port was specified in the FANCYLOG_SERVER_PORT env var. this takes precedence - if 'FANCYLOG_SERVER_PORT' in os.environ: - port = int(os.environ['FANCYLOG_SERVER_PORT']) - port = int(port) + port = os.environ.get('FANCYLOG_SERVER_PORT', port) - logToUDP(server, port) + logToUDP(server, int(port)) _default_logTo = logToUDP else: # log to screen by default diff --git a/easybuild/main.py b/easybuild/main.py index 73550e3998..047e4938ee 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -736,8 +736,7 @@ def prepare_main(args=None, logfile=None, testing=None): # if $CDPATH is set, unset it, it'll only cause trouble... # see https://github.com/easybuilders/easybuild-framework/issues/2944 - if 'CDPATH' in os.environ: - del os.environ['CDPATH'] + os.environ.pop('CDPATH', None) # When EB is run via `exec` the special bash variable $_ is not set # So emulate this here to allow (module) scripts depending on that to work diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index d68755ff24..f6055bccc1 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -83,9 +83,9 @@ def setvar(key, value, verbose=True): :param verbose: include message in dry run output for defining this environment variable """ - if key in os.environ: + try: oldval_info = "previous value: '%s'" % os.environ[key] - else: + except KeyError: oldval_info = "previously undefined" # os.putenv() is not necessary. os.environ will call this. os.environ[key] = value diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index f2d74c2137..b651502e95 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -307,9 +307,9 @@ def check_module_function(self, allow_mismatch=False, regex=None): """Check whether selected module tool matches 'module' function definition.""" if self.testing: # grab 'module' function definition from environment if it's there; only during testing - if 'module' in os.environ: + try: out, ec = os.environ['module'], 0 - else: + except KeyError: out, ec = None, 1 else: cmd = "type module" @@ -1561,9 +1561,7 @@ def get_software_root(name, with_env_var=False): """ env_var = get_software_root_env_var_name(name) - root = None - if env_var in os.environ: - root = os.getenv(env_var) + root = os.getenv(env_var) if with_env_var: res = (root, env_var) @@ -1631,9 +1629,7 @@ def get_software_version(name): """ env_var = get_software_version_env_var_name(name) - version = None - if env_var in os.environ: - version = os.getenv(env_var) + version = os.getenv(env_var) return version diff --git a/test/framework/environment.py b/test/framework/environment.py index 9a81e17486..18359fc776 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -90,8 +90,7 @@ def test_modify_env(self): # prepare test environment first: # keys in new_env should not be set yet, keys in old_env are expected to be set for key in new_env_vars: - if key in os.environ: - del os.environ[key] + os.environ.pop(key, None) for key in old_env_vars: os.environ[key] = old_env_vars[key] diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3b879bef85..80ee733ded 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1505,8 +1505,7 @@ def test_find_flexlm_license(self): lic_server = '1234@example.license.server' # make test robust against environment in which $LM_LICENSE_FILE is defined - if 'LM_LICENSE_FILE' in os.environ: - del os.environ['LM_LICENSE_FILE'] + os.environ.pop('LM_LICENSE_FILE', None) # default return value self.assertEqual(ft.find_flexlm_license(), ([], None)) @@ -2630,8 +2629,7 @@ def test_find_eb_script(self): """Test find_eb_script function.""" # make sure $EB_SCRIPT_PATH is not set already (used as fallback mechanism in find_eb_script) - if 'EB_SCRIPT_PATH' in os.environ: - del os.environ['EB_SCRIPT_PATH'] + os.environ.pop('EB_SCRIPT_PATH', None) self.assertExists(ft.find_eb_script('rpath_args.py')) self.assertExists(ft.find_eb_script('rpath_wrapper_template.sh.in')) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index a0c6131140..45d7402523 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -900,8 +900,7 @@ def test_getenv_cmd(self): if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: # can't have $LMOD_QUIET set when testing with Tcl syntax, # otherwise we won't get the output produced by the test module file... - if 'LMOD_QUIET' in os.environ: - del os.environ['LMOD_QUIET'] + os.environ.pop('LMOD_QUIET', None) self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME')) diff --git a/test/framework/modules.py b/test/framework/modules.py index 4a8a6b15df..10ae419294 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -99,8 +99,7 @@ def test_run_module(self): testdir = os.path.dirname(os.path.abspath(__file__)) for key in ['EBROOTGCC', 'EBROOTOPENMPI', 'EBROOTOPENBLAS']: - if key in os.environ: - del os.environ[key] + os.environ.pop(key, None) # arguments can be passed in two ways: multiple arguments, or just 1 list argument self.modtool.run_module('load', 'GCC/6.4.0-2.28') @@ -447,8 +446,7 @@ def test_load(self): self.assertEqual(self.modtool.loaded_modules()[-1], 'GCC/6.4.0-2.28') # set things up for checking that GCC does *not* get reloaded when requested - if 'EBROOTGCC' in os.environ: - del os.environ['EBROOTGCC'] + os.environ.pop('EBROOTGCC', None) self.modtool.load(['OpenMPI/2.1.2-GCC-6.4.0-2.28']) if isinstance(self.modtool, Lmod): # order of loaded modules only changes with Lmod @@ -1025,8 +1023,7 @@ def test_modules_tool_stateless(self): init_config() # make sure $LMOD_DEFAULT_MODULEPATH, since Lmod picks it up and tweaks $MODULEPATH to match it - if 'LMOD_DEFAULT_MODULEPATH' in os.environ: - del os.environ['LMOD_DEFAULT_MODULEPATH'] + os.environ.pop('LMOD_DEFAULT_MODULEPATH', None) self.reset_modulepath([os.path.join(self.test_prefix, 'Core')]) @@ -1048,8 +1045,7 @@ def test_modules_tool_stateless(self): self.modtool.load(['OpenMPI/2.1.2']) self.modtool.purge() - if 'LMOD_DEFAULT_MODULEPATH' in os.environ: - del os.environ['LMOD_DEFAULT_MODULEPATH'] + os.environ.pop('LMOD_DEFAULT_MODULEPATH', None) # reset $MODULEPATH, obtain new ModulesTool instance, # which should not remember anything w.r.t. previous $MODULEPATH value diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 59c6872b93..894bdce009 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -200,8 +200,7 @@ def tearDown(self): if self.orig_module is not None: os.environ['module'] = self.orig_module else: - if 'module' in os.environ: - del os.environ['module'] + os.environ.pop('module', None) def suite(): diff --git a/test/framework/options.py b/test/framework/options.py index f3173dbb87..06cb14eec4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3154,8 +3154,7 @@ def test_show_default_configfiles(self): home = os.environ['HOME'] for envvar in ['XDG_CONFIG_DIRS', 'XDG_CONFIG_HOME']: - if envvar in os.environ: - del os.environ[envvar] + os.environ.pop(envvar, None) reload(easybuild.tools.options) args = [ @@ -4957,8 +4956,7 @@ def test_show_config_cfg_levels(self): """Test --show-config in relation to how configuring across multiple configuration levels interacts with it.""" # make sure default module syntax is used - if 'EASYBUILD_MODULE_SYNTAX' in os.environ: - del os.environ['EASYBUILD_MODULE_SYNTAX'] + os.environ.pop('EASYBUILD_MODULE_SYNTAX', None) # configuring --modules-tool and --module-syntax on different levels should NOT cause problems # cfr. bug report https://github.com/easybuilders/easybuild-framework/issues/2564 @@ -4984,8 +4982,7 @@ def test_modules_tool_vs_syntax_check(self): """Verify that check for modules tool vs syntax works.""" # make sure default module syntax is used - if 'EASYBUILD_MODULE_SYNTAX' in os.environ: - del os.environ['EASYBUILD_MODULE_SYNTAX'] + os.environ.pop('EASYBUILD_MODULE_SYNTAX', None) # using EnvironmentModulesC modules tool with default module syntax (Lua) is a problem os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC' @@ -6764,8 +6761,7 @@ def test_easystack_opts(self): mod_ext = '.lua' if get_module_syntax() == 'Lua' else '' # make sure that $EBROOTLIBTOY is not defined - if 'EBROOTLIBTOY' in os.environ: - del os.environ['EBROOTLIBTOY'] + os.environ.pop('EBROOTLIBTOY', None) # libtoy module should be installed, module file should at least set EBROOTLIBTOY mod_dir = os.path.join(self.test_installpath, 'modules', 'all') diff --git a/test/framework/robot.py b/test/framework/robot.py index 4f226bae35..89e24d008f 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -128,8 +128,7 @@ def tearDown(self): config.modules_tool = ORIG_MODULES_TOOL ecec.modules_tool = ORIG_ECEC_MODULES_TOOL if ORIG_MODULE_FUNCTION is None: - if 'module' in os.environ: - del os.environ['module'] + os.environ.pop('module', None) else: os.environ['module'] = ORIG_MODULE_FUNCTION self.modtool = self.orig_modtool diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index aacde3baf7..2d5511776d 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -181,8 +181,7 @@ def test_toolchain_prepare_sysroot(self): # clean environment self.unset_compiler_env_vars() - if 'PKG_CONFIG_PATH' in os.environ: - del os.environ['PKG_CONFIG_PATH'] + os.environ.pop('PKG_CONFIG_PATH', None) self.assertEqual(os.getenv('PKG_CONFIG_PATH'), None) @@ -227,8 +226,7 @@ def unset_compiler_env_vars(self): env_vars.extend(['OMPI_%s' % x for x in comp_env_vars]) for key in env_vars: - if key in os.environ: - del os.environ[key] + os.environ.pop(key, None) def test_toolchain_compiler_env_vars(self): """Test whether environment variables for compilers are defined by toolchain mechanism.""" @@ -345,8 +343,7 @@ def test_toolchain_compiler_env_vars(self): init_config(build_options={'minimal_build_env': 'CC:gcc,CXX:g++,CFLAGS:-O2,CXXFLAGS:-O3 -g,FC:gfortan'}) for key in ['CFLAGS', 'CXXFLAGS', 'FC']: - if key in os.environ: - del os.environ[key] + os.environ.pop(key, None) self.mock_stderr(True) self.mock_stdout(True) @@ -2270,8 +2267,7 @@ def test_rpath_args_script(self): """Test rpath_args.py script""" # $LIBRARY_PATH affects result of rpath_args.py, so make sure it's not set - if 'LIBRARY_PATH' in os.environ: - del os.environ['LIBRARY_PATH'] + os.environ.pop('LIBRARY_PATH', None) script = find_eb_script('rpath_args.py') diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9ccb3c08df..faac2a7e76 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -344,8 +344,7 @@ def test_toy_tweaked(self): expected += "oh hai!" # setting $LMOD_QUIET results in suppression of printed message with Lmod & module files in Tcl syntax - if 'LMOD_QUIET' in os.environ: - del os.environ['LMOD_QUIET'] + os.environ.pop('LMOD_QUIET', None) self.modtool.use(os.path.join(self.test_installpath, 'modules', 'all')) out = self.modtool.run_module('load', 'toy/0.0-tweaked', return_output=True) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 37a9b29318..6a9abed4a5 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -207,8 +207,7 @@ def disallow_deprecated_behaviour(self): def allow_deprecated_behaviour(self): """Restore EasyBuild version to what it was originally, to allow triggering deprecated behaviour.""" - if 'EASYBUILD_DEPRECATED' in os.environ: - del os.environ['EASYBUILD_DEPRECATED'] + os.environ.pop('EASYBUILD_DEPRECATED', None) eb_build_log.CURRENT_VERSION = self.orig_current_version @contextmanager @@ -278,8 +277,7 @@ def reset_modulepath(self, modpaths): # make very sure $MODULEPATH is totally empty # some paths may be left behind, e.g. when they contain environment variables # example: "module unuse Modules/$MODULE_VERSION/modulefiles" may not yield the desired result - if 'MODULEPATH' in os.environ: - del os.environ['MODULEPATH'] + os.environ.pop('MODULEPATH', None) for modpath in modpaths: self.modtool.add_module_path(modpath, set_mod_paths=False) self.modtool.set_mod_paths() From a748eaf2e8f65f73b0e8557162567143de09f11a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 11:11:07 +0100 Subject: [PATCH 022/430] Show empty description for license instead of "None" --- easybuild/tools/docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4fcf1ad466..1d049a19c1 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -289,7 +289,7 @@ def avail_easyconfig_licenses_md(): lics = sorted(EASYCONFIG_LICENSES_DICT.items()) table_values = [ ["``%s``" % lic().name for _, lic in lics], - [lic().description for _, lic in lics], + [lic().description or '' for _, lic in lics], ["``%s``" % lic().version for _, lic in lics], ] From 6e919e4ece21b8175d0dee9f8c0fdce8f6f1dcbc Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 10:03:19 +0100 Subject: [PATCH 023/430] Use more performant and concease dict construction The pattern `dict([(key, value) for key, value in (construction)])` is very verbose and creates an intermediate list of tuples. Replace by `{key: value for key, value in (construction)}` to let Python core handle it without unncessary intermediates. --- easybuild/base/generaloption.py | 5 +++-- easybuild/framework/easyconfig/default.py | 2 +- easybuild/framework/easyconfig/easyconfig.py | 16 ++++++++-------- .../easyconfig/format/pyheaderconfigobj.py | 6 +++--- easybuild/framework/easyconfig/format/version.py | 2 +- easybuild/framework/easyconfig/templates.py | 2 +- easybuild/toolchains/mpi/craympich.py | 2 +- easybuild/toolchains/mpi/fujitsumpi.py | 2 +- easybuild/toolchains/mpi/mpich.py | 2 +- easybuild/toolchains/mpi/mpitrampoline.py | 2 +- easybuild/toolchains/mpi/openmpi.py | 2 +- easybuild/tools/config.py | 4 ++-- easybuild/tools/docs.py | 7 ++----- easybuild/tools/environment.py | 2 +- easybuild/tools/job/backend.py | 2 +- easybuild/tools/job/pbs_python.py | 2 +- easybuild/tools/module_generator.py | 2 +- .../module_naming_scheme/hierarchical_mns.py | 2 +- .../tools/module_naming_scheme/utilities.py | 2 +- easybuild/tools/modules.py | 8 ++++---- easybuild/tools/options.py | 2 +- easybuild/tools/package/utilities.py | 2 +- easybuild/tools/repository/repository.py | 2 +- test/framework/easyconfig.py | 2 +- test/framework/filetools.py | 2 +- 25 files changed, 41 insertions(+), 43 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 51f7daafe3..9bb5e32e50 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -199,7 +199,8 @@ class ExtOption(CompleterOption): ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']] - TYPE_CHECKER = dict([(x, check_str_list_tuple) for x in TYPE_STRLIST] + list(Option.TYPE_CHECKER.items())) + TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST} + TYPE_CHECKER.update(Option.TYPE_CHECKER) TYPES = tuple(TYPE_STRLIST + list(Option.TYPES)) BOOLEAN_ACTIONS = ('store_true', 'store_false',) + EXTOPTION_LOG @@ -807,7 +808,7 @@ def get_env_options(self): epilogprefixtxt += "eg. --some-opt is same as setting %(prefix)s_SOME_OPT in the environment." self.epilog.append(epilogprefixtxt % {'prefix': self.envvar_prefix}) - candidates = dict([(k, v) for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)]) + candidates = {k: v for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)} for opt in self._get_all_options(): if opt._long_opts is None: diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dd91229d1e..fb6e5fc657 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -60,7 +60,7 @@ # we use a tuple here so we can sort them based on the numbers CATEGORY_NAMES = ['BUILD', 'CUSTOM', 'DEPENDENCIES', 'EXTENSIONS', 'FILEMANAGEMENT', 'HIDDEN', 'LICENSE', 'MANDATORY', 'MODULES', 'OTHER', 'TOOLCHAIN'] -ALL_CATEGORIES = dict((name, eval(name)) for name in CATEGORY_NAMES) +ALL_CATEGORIES = {name: eval(name) for name in CATEGORY_NAMES} # List of tuples. Each tuple has the following format (key, [default, help text, category]) DEFAULT_CONFIG = { diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ff4ed3562c..9cbbd44094 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -303,7 +303,7 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): """ # obtain list of all possible subtoolchains _, all_tc_classes = search_toolchain('') - subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes) + subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes} optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)) composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1) @@ -985,7 +985,7 @@ def filter_hidden_deps(self): faulty_deps = [] # obtain reference to original lists, so their elements can be changed in place - deps = dict([(key, self.get_ref(key)) for key in ['dependencies', 'builddependencies', 'hiddendependencies']]) + deps = {key: self.get_ref(key) for key in ('dependencies', 'builddependencies', 'hiddendependencies')} if 'builddependencies' in self.iterate_options: deplists = copy.deepcopy(deps['builddependencies']) @@ -1229,11 +1229,11 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals # templated values should be dumped unresolved with self.disable_templating(): # build dict of default values - default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG]) - default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options])) + default_values = {key: DEFAULT_CONFIG[key][0] for key in DEFAULT_CONFIG} + default_values.update({key: self.extra_options[key][0] for key in self.extra_options}) self.generate_template_values() - templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS]) + templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS} # create reverse map of templates, to inject template values where possible # longer template values are considered first, shorter template keys get preference over longer ones @@ -1520,7 +1520,6 @@ def _parse_dependency(self, dep, hidden=False, build_only=False): # convert tuple to string otherwise python might complain about the formatting self.log.debug("Parsing %s as a dependency" % str(dep)) - attr = ['name', 'version', 'versionsuffix', 'toolchain'] dependency = { # full/short module names 'full_mod_name': None, @@ -1576,6 +1575,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False): raise EasyBuildError("Incorrect external dependency specification: %s", dep) else: # non-external dependency: tuple (or list) that specifies name/version(/versionsuffix(/toolchain)) + attr = ('name', 'version', 'versionsuffix', 'toolchain') dependency.update(dict(zip(attr, dep))) else: @@ -2049,7 +2049,7 @@ def resolve_template(value, tmpl_dict): elif isinstance(value, tuple): value = tuple(resolve_template(list(value), tmpl_dict)) elif isinstance(value, dict): - value = dict((resolve_template(k, tmpl_dict), resolve_template(v, tmpl_dict)) for k, v in value.items()) + value = {resolve_template(k, tmpl_dict): resolve_template(v, tmpl_dict) for k, v in value.items()} return value @@ -2256,7 +2256,7 @@ def verify_easyconfig_filename(path, specs, parsed_ec=None): for ec in ecs: found_fullver = det_full_ec_version(ec['ec']) if ec['ec']['name'] != specs['name'] or found_fullver != fullver: - subspec = dict((key, specs[key]) for key in ['name', 'toolchain', 'version', 'versionsuffix']) + subspec = {key: specs[key] for key in ('name', 'toolchain', 'version', 'versionsuffix')} error_msg = "Contents of %s does not match with filename" % path error_msg += "; expected filename based on contents: %s-%s.eb" % (ec['ec']['name'], found_fullver) error_msg += "; expected (relevant) parameters based on filename %s: %s" % (os.path.basename(path), subspec) diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index b9fd31bf92..72454efd7b 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -51,9 +51,9 @@ def build_easyconfig_constants_dict(): """Make a dictionary with all constants that can be used""" all_consts = [ - ('TEMPLATE_CONSTANTS', dict([(x[0], x[1]) for x in TEMPLATE_CONSTANTS])), - ('EASYCONFIG_CONSTANTS', dict([(key, val[0]) for key, val in EASYCONFIG_CONSTANTS.items()])), - ('EASYCONFIG_LICENSES', dict([(klass().name, name) for name, klass in EASYCONFIG_LICENSES_DICT.items()])), + ('TEMPLATE_CONSTANTS', {x[0]: x[1] for x in TEMPLATE_CONSTANTS}), + ('EASYCONFIG_CONSTANTS', {key: val[0] for key, val in EASYCONFIG_CONSTANTS.items()}), + ('EASYCONFIG_LICENSES', {klass().name: name for name, klass in EASYCONFIG_LICENSES_DICT.items()}), ] err = [] const_dict = {} diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index fe3b1a2316..ce20ca94a3 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -69,7 +69,7 @@ class VersionOperator(object): '<': op.lt, '<=': op.le, } - REVERSE_OPERATOR_MAP = dict([(v, k) for k, v in OPERATOR_MAP.items()]) + REVERSE_OPERATOR_MAP = {v: k for k, v in OPERATOR_MAP.items()} INCLUDE_OPERATORS = ['==', '>=', '<='] # these operators *include* the (version) boundary ORDERED_OPERATORS = ['==', '>', '>=', '<', '<='] # ordering by strictness OPERATOR_FAMILIES = [['>', '>='], ['<', '<=']] # similar operators diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index aed8db6af4..bce2d139f1 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -258,7 +258,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # step 2: define *ver and *shortver templates if TEMPLATE_SOFTWARE_VERSIONS: - name_to_prefix = dict((name.lower(), pref) for name, pref in TEMPLATE_SOFTWARE_VERSIONS) + name_to_prefix = {name.lower(): pref for name, pref in TEMPLATE_SOFTWARE_VERSIONS} deps = config.get('dependencies', []) # also consider build dependencies for *ver and *shortver templates; diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py index 4c62c45519..7b071ad8da 100644 --- a/easybuild/toolchains/mpi/craympich.py +++ b/easybuild/toolchains/mpi/craympich.py @@ -53,7 +53,7 @@ class CrayMPICH(Mpi): MPI_COMPILER_MPIFC = CrayPECompiler.COMPILER_FC # no MPI wrappers, so no need to specify serial compiler - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} def _set_mpi_compiler_variables(self): """Set the MPI compiler variables""" diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py index f81e1d66f0..704eab1742 100644 --- a/easybuild/toolchains/mpi/fujitsumpi.py +++ b/easybuild/toolchains/mpi/fujitsumpi.py @@ -45,7 +45,7 @@ class FujitsuMPI(Mpi): MPI_TYPE = TC_CONSTANT_MPI_TYPE_OPENMPI # OpenMPI reads from CC etc env variables - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} MPI_LINK_INFO_OPTION = '-showme:link' diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py index b0c42e57d8..45b8580799 100644 --- a/easybuild/toolchains/mpi/mpich.py +++ b/easybuild/toolchains/mpi/mpich.py @@ -56,7 +56,7 @@ class Mpich(Mpi): MPI_COMPILER_MPIFC = None # clear MPI wrapper command options - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} def _set_mpi_compiler_variables(self): """Set the MPICH_{CC, CXX, F77, F90, FC} variables.""" diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py index 5af054661c..f3b8a332ac 100644 --- a/easybuild/toolchains/mpi/mpitrampoline.py +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -53,7 +53,7 @@ class MPItrampoline(Mpi): MPI_COMPILER_MPIFC = None # MPItrampoline reads from CC etc env variables - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} MPI_LINK_INFO_OPTION = '-showme:link' diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index 630262f48b..787fefd429 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -61,7 +61,7 @@ class OpenMPI(Mpi): MPI_COMPILER_MPIFC = None # OpenMPI reads from CC etc env variables - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} MPI_LINK_INFO_OPTION = '-showme:link' diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ee8eb5fc53..09d18dbaf3 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -574,7 +574,7 @@ def init_build_options(build_options=None, cmdline_options=None): cmdline_options.accept_eula_for = cmdline_options.accept_eula cmdline_build_option_names = [k for ks in BUILD_OPTIONS_CMDLINE.values() for k in ks] - active_build_options.update(dict([(key, getattr(cmdline_options, key)) for key in cmdline_build_option_names])) + active_build_options.update({key: getattr(cmdline_options, key) for key in cmdline_build_option_names}) # other options which can be derived but have no perfectly matching cmdline option active_build_options.update({ 'check_osdeps': not cmdline_options.ignore_osdeps, @@ -597,7 +597,7 @@ def init_build_options(build_options=None, cmdline_options=None): for opt in build_options_by_default[default]: bo[opt] = [] else: - bo.update(dict([(opt, default) for opt in build_options_by_default[default]])) + bo.update({opt: default for opt in build_options_by_default[default]}) bo.update(active_build_options) # BuildOptions is a singleton, so any future calls to BuildOptions will yield the same instance diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..57671a8b99 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1028,12 +1028,9 @@ def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') - # filter deprecated 'dummy' toolchain - all_tcs = [x for x in all_tcs if x.NAME != DUMMY_TOOLCHAIN_NAME] - all_tcs_names = [x.NAME for x in all_tcs] - # start with dict that maps toolchain name to corresponding subclass of Toolchain - tcs = dict(zip(all_tcs_names, all_tcs)) + # filter deprecated 'dummy' toolchain + tcs = {tc.NAME: tc for tc in all_tcs if tc.NAME != DUMMY_TOOLCHAIN_NAME} for tcname in sorted(tcs): tcc = tcs[tcname] diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index d68755ff24..115a08891d 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -136,7 +136,7 @@ def read_environment(env_vars, strict=False): :param env_vars: a dict with key a name, value a environment variable name :param strict: boolean, if True enforces that all specified environment variables are found """ - result = dict([(k, os.environ.get(v)) for k, v in env_vars.items() if v in os.environ]) + result = {k: os.environ.get(v) for k, v in env_vars.items() if v in os.environ} if not len(env_vars) == len(result): missing = ','.join(["%s / %s" % (k, v) for k, v in env_vars.items() if k not in result]) diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py index c92ead6a58..0b1f69eb08 100644 --- a/easybuild/tools/job/backend.py +++ b/easybuild/tools/job/backend.py @@ -104,7 +104,7 @@ def avail_job_backends(check_usable=True): Return all known job execution backends. """ import_available_modules('easybuild.tools.job') - class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)]) + class_dict = {x.__name__: x for x in get_subclasses(JobBackend)} return class_dict diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index a4a020988e..1ad13738ba 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -484,7 +484,7 @@ def info(self, types=None): # only expect to have a list with one element j = jobs[0] # convert attribs into useable dict - job_details = dict([(attrib.name, attrib.value) for attrib in j.attribs]) + job_details = {attrib.name: attrib.value for attrib in j.attribs} # manually set 'id' attribute job_details['id'] = j.name self.log.debug("Found jobinfo %s" % job_details) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 0c19b93c8b..723753e31a 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -59,7 +59,7 @@ def avail_module_generators(): """ Return all known module syntaxes. """ - return dict([(k.SYNTAX, k) for k in get_subclasses(ModuleGenerator)]) + return {k.SYNTAX: k for k in get_subclasses(ModuleGenerator)} def module_generator(app, fake=False): diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 9f2e026eb8..0de0eb5107 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -120,7 +120,7 @@ def det_toolchain_compilers_name_version(self, tc_comps): res = None else: if len(tc_comps) > 0 and tc_comps[0]: - comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps]) + comp_versions = {comp['name']: self.det_full_version(comp) for comp in tc_comps} comp_names = comp_versions.keys() key = ','.join(sorted(comp_names)) if key in COMP_NAME_VERSION_TEMPLATES: diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index e46609d22b..2878cbf1a1 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -86,7 +86,7 @@ def avail_module_naming_schemes(): import_available_modules('easybuild.tools.module_naming_scheme') # construct name-to-class dict of available module naming scheme - avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)]) + avail_mnss = {x.__name__: x for x in get_subclasses(ModuleNamingScheme)} return avail_mnss diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index f2d74c2137..a6a81781dc 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -851,7 +851,7 @@ def run_module(self, *args, **kwargs): # this needs to be taken into account when updating the environment via produced output, see below # keep track of current values of select env vars, so we can correct the adjusted values below - prev_ld_values = dict([(key, os.environ.get(key, '').split(os.pathsep)[::-1]) for key in LD_ENV_VAR_KEYS]) + prev_ld_values = {key: os.environ.get(key, '').split(os.pathsep)[::-1] for key in LD_ENV_VAR_KEYS} # Change the environment try: @@ -1126,7 +1126,7 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, if modpath_exts is None: # only retain dependencies that have a non-empty lists of $MODULEPATH extensions - modpath_exts = dict([(k, v) for k, v in self.modpath_extensions_for(deps).items() if v]) + modpath_exts = {k: v for k, v in self.modpath_extensions_for(deps).items() if v} self.log.debug("Non-empty lists of module path extensions for dependencies: %s" % modpath_exts) mods_to_top = [] @@ -1157,7 +1157,7 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, path = mods_to_top[:] if mods_to_top: # remove retained dependencies from the list, since we're climbing up the module tree - remaining_modpath_exts = dict([m for m in modpath_exts.items() if not m[0] in mods_to_top]) + remaining_modpath_exts = {m: v for m, v in modpath_exts.items() if m not in mods_to_top} self.log.debug("Path to top from %s extended to %s, so recursing to find way to the top", mod_name, mods_to_top) @@ -1662,7 +1662,7 @@ def avail_modules_tools(): """ Return all known modules tools. """ - class_dict = dict([(x.__name__, x) for x in get_subclasses(ModulesTool)]) + class_dict = {x.__name__: x for x in get_subclasses(ModulesTool)} # filter out legacy Modules class if 'Modules' in class_dict: del class_dict['Modules'] diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index b840bc4d53..3364256957 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -244,7 +244,7 @@ def __init__(self, *args, **kwargs): # update or define go_configfiles_initenv in named arguments to pass to parent constructor go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) for section, constants in self.go_cfg_constants.items(): - constants = dict([(name, value) for (name, (value, _)) in constants.items()]) + constants = {name: value for (name, (value, _)) in constants.items()} go_cfg_initenv.setdefault(section, {}).update(constants) super(EasyBuildOptions, self).__init__(*args, **kwargs) diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 087084c687..8de28e5e5b 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -58,7 +58,7 @@ def avail_package_naming_schemes(): They are loaded from the easybuild.package.package_naming_scheme namespace """ import_available_modules('easybuild.tools.package.package_naming_scheme') - class_dict = dict([(x.__name__, x) for x in get_subclasses(PackageNamingScheme)]) + class_dict = {x.__name__: x for x in get_subclasses(PackageNamingScheme)} return class_dict diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index cbd33ab654..effa5ae0bf 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -146,7 +146,7 @@ def avail_repositories(check_useable=True): """ import_available_modules('easybuild.tools.repository') - class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable]) + class_dict = {x.__name__: x for x in get_subclasses(Repository) if x.USABLE or not check_useable} if 'FileRepository' not in class_dict: raise EasyBuildError("avail_repositories: FileRepository missing from list of repositories") diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index dda5c0a05b..219b481b69 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3621,7 +3621,7 @@ def test_resolve_template(self): def test_det_subtoolchain_version(self): """Test det_subtoolchain_version function""" _, all_tc_classes = search_toolchain('') - subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes) + subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes} optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)) current_tc = {'name': 'fosscuda', 'version': '2018a'} diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3b879bef85..cf28cbc257 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -320,7 +320,7 @@ def test_checksums(self): self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum) # make sure faulty checksums are reported - broken_checksums = dict([(typ, val[:-3] + 'foo') for (typ, val) in known_checksums.items()]) + broken_checksums = {typ: (val[:-3] + 'foo') for (typ, val) in known_checksums.items()} for checksum_type, checksum in broken_checksums.items(): self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum) self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum))) From 13191968a1b43436c5df63d8cd52edc8d503560c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 13:41:47 +0100 Subject: [PATCH 024/430] Remove typo Co-authored-by: Kenneth Hoste --- .../tools/module_naming_scheme/test_module_naming_scheme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 02934e5745..5d8dec26c8 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -64,6 +64,6 @@ def det_module_symlink_paths(self, ec): def is_short_modname_for(self, modname, name): """ - Determine whether the specified (short) module name is a moduleq for software with the specified name. + Determine whether the specified (short) module name is a module for software with the specified name. """ return modname.find(name) != -1 From cd1c1f5abc52bdfa6b9618fa8980f0f184a19e93 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 11:08:04 +0100 Subject: [PATCH 025/430] Fix compatibility with Python2 --- easybuild/tools/modules.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index a6a81781dc..8674c4350b 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -851,7 +851,9 @@ def run_module(self, *args, **kwargs): # this needs to be taken into account when updating the environment via produced output, see below # keep track of current values of select env vars, so we can correct the adjusted values below - prev_ld_values = {key: os.environ.get(key, '').split(os.pathsep)[::-1] for key in LD_ENV_VAR_KEYS} + # Identical to `{key: os.environ.get(key, '').split(os.pathsep)[::-1] for key in LD_ENV_VAR_KEYS}` + # but Python 2 treats that as a local function and refused the `exec` below + prev_ld_values = dict([(key, os.environ.get(key, '').split(os.pathsep)[::-1]) for key in LD_ENV_VAR_KEYS]) # Change the environment try: From cf02d8333a67900f7d04e96522232d8bef5760ed Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 13:40:56 +0100 Subject: [PATCH 026/430] Remove parens from dict construction for consistency --- easybuild/tools/options.py | 2 +- test/framework/filetools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 3364256957..75fbf65b9b 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -244,7 +244,7 @@ def __init__(self, *args, **kwargs): # update or define go_configfiles_initenv in named arguments to pass to parent constructor go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) for section, constants in self.go_cfg_constants.items(): - constants = {name: value for (name, (value, _)) in constants.items()} + constants = {name: value for name, (value, _) in constants.items()} go_cfg_initenv.setdefault(section, {}).update(constants) super(EasyBuildOptions, self).__init__(*args, **kwargs) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index cf28cbc257..d15d0df7c4 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -320,7 +320,7 @@ def test_checksums(self): self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum) # make sure faulty checksums are reported - broken_checksums = {typ: (val[:-3] + 'foo') for (typ, val) in known_checksums.items()} + broken_checksums = {typ: (val[:-3] + 'foo') for typ, val in known_checksums.items()} for checksum_type, checksum in broken_checksums.items(): self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum) self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum))) From 3830181bb4a6fad4f8d3f71f884961345fc47eae Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 16:13:31 +0100 Subject: [PATCH 027/430] Cleanup uses of `getattr` and `hasattr` `hasattr` is basically calling `getattr` and checking for `AttributeError`. It also supports a default value in case the attribute is not found. Make use of that default value and catch the exception directly instead of checking `hasattr` where that makes sense. --- easybuild/base/fancylogger.py | 2 +- easybuild/base/generaloption.py | 2 +- easybuild/base/optcomplete.py | 21 ++++++++-------- .../easyconfig/format/pyheaderconfigobj.py | 11 +++++---- easybuild/tools/configobj.py | 2 +- easybuild/tools/docs.py | 7 +++--- easybuild/tools/github.py | 4 ++-- easybuild/tools/systemtools.py | 4 ++-- easybuild/tools/toolchain/linalg.py | 10 ++++---- easybuild/tools/toolchain/toolchain.py | 24 +++++++++++-------- easybuild/tools/toolchain/utilities.py | 11 +++++---- test/framework/systemtools.py | 7 ++---- 12 files changed, 53 insertions(+), 52 deletions(-) diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index b5a63477c6..93582b0cf1 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -286,7 +286,7 @@ def makeRecord(self, name, level, pathname, lineno, msg, args, excinfo, func=Non overwrite make record to use a fancy record (with more options) """ logrecordcls = logging.LogRecord - if hasattr(self, 'fancyrecord') and self.fancyrecord: + if getattr(self, 'fancyrecord', None): logrecordcls = FancyLogRecord try: new_msg = str(msg) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 51f7daafe3..49b1168008 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1031,7 +1031,7 @@ def main_options(self): # make_init is deprecated if hasattr(self, 'make_init'): self.log.debug('main_options: make_init is deprecated. Rename function to main_options.') - getattr(self, 'make_init')() + self.make_init() else: # function names which end with _options and do not start with main or _ reg_main_options = re.compile("^(?!_|main).*_options$") diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index 9a2bc8a127..3fa75a6635 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -513,14 +513,15 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete if option: if option.nargs > 0: optarg = True - if hasattr(option, 'completer'): + try: completer = option.completer - elif option.choices: - completer = ListCompleter(option.choices) - elif option.type in ('string',): - completer = opt_completer - else: - completer = NoneCompleter() + except AttributeError: + if option.choices: + completer = ListCompleter(option.choices) + elif option.type in ('string',): + completer = opt_completer + else: + completer = NoneCompleter() # Warn user at least, it could help him figure out the problem. elif hasattr(option, 'completer'): msg = "Error: optparse option with a completer does not take arguments: %s" % (option) @@ -616,11 +617,9 @@ class CmdComplete(object): def autocomplete(self, completer=None): parser = OPTIONPARSER_CLASS(self.__doc__.strip()) if hasattr(self, 'addopts'): - fnc = getattr(self, 'addopts') - fnc(parser) + self.addopts(parser) - if hasattr(self, 'completer'): - completer = getattr(self, 'completer') + completer = getattr(self, 'completer', completer) return autocomplete(parser, completer) diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index b9fd31bf92..37e14b529e 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -233,12 +233,13 @@ def pyheader_env(self): current_builtins = globals()['__builtins__'] builtins = {} for name in self.PYHEADER_ALLOWED_BUILTINS: - if hasattr(current_builtins, name): + try: builtins[name] = getattr(current_builtins, name) - elif isinstance(current_builtins, dict) and name in current_builtins: - builtins[name] = current_builtins[name] - else: - self.log.warning('No builtin %s found.' % name) + except AttributeError: + if isinstance(current_builtins, dict) and name in current_builtins: + builtins[name] = current_builtins[name] + else: + self.log.warning('No builtin %s found.' % name) global_vars['__builtins__'] = builtins self.log.debug("Available builtins: %s" % global_vars['__builtins__']) diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index 48c7dd348d..6ac667b5bd 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -1254,7 +1254,7 @@ def set_section(in_section, this_section): self.configspec = None return - elif getattr(infile, 'read', MISSING) is not MISSING: + elif hasattr(infile, 'read'): # This supports file like objects infile = infile.read() or [] # needs splitting into lines - but needs doing *after* decoding diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..dc707466ad 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1182,10 +1182,9 @@ def avail_toolchain_opts(name, output_format=FORMAT_TXT): tc_dict = {} for cst in ['COMPILER_SHARED_OPTS', 'COMPILER_UNIQUE_OPTS', 'MPI_SHARED_OPTS', 'MPI_UNIQUE_OPTS']: - if hasattr(tc, cst): - opts = getattr(tc, cst) - if opts is not None: - tc_dict.update(opts) + opts = getattr(tc, cst, None) + if opts is not None: + tc_dict.update(opts) return generate_doc('avail_toolchain_opts_%s' % output_format, [name, tc_dict]) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 983dc977c4..ca013743d7 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -719,9 +719,9 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa raise EasyBuildError("Fetching branch '%s' from remote %s failed: empty result", branch_name, origin) # git checkout -b ; git pull - if hasattr(origin.refs, branch_name): + try: origin_branch = getattr(origin.refs, branch_name) - else: + except AttributeError: raise EasyBuildError("Branch '%s' not found at %s", branch_name, github_url) _log.debug("Checking out branch '%s' from remote %s", branch_name, github_url) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 78ba312203..59b8728454 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1340,8 +1340,8 @@ def det_pypkg_version(pkg_name, imported_pkg, import_name=None): except pkg_resources.DistributionNotFound as err: _log.debug("%s Python package not found: %s", pkg_name, err) - if version is None and hasattr(imported_pkg, '__version__'): - version = imported_pkg.__version__ + if version is None: + version = getattr(imported_pkg, '__version__', None) return version diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index b7ce694ff7..907993571c 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -212,9 +212,9 @@ def _set_blacs_variables(self): """Set BLACS related variables""" lib_map = {} - if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: + if getattr(self, 'BLAS_LIB_MAP', None) is not None: lib_map.update(self.BLAS_LIB_MAP) - if hasattr(self, 'BLACS_LIB_MAP') and self.BLACS_LIB_MAP is not None: + if getattr(self, 'BLACS_LIB_MAP', None) is not None: lib_map.update(self.BLACS_LIB_MAP) # BLACS @@ -254,11 +254,11 @@ def _set_scalapack_variables(self): raise EasyBuildError("_set_blas_variables: SCALAPACK_LIB not set") lib_map = {} - if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: + if getattr(self, 'BLAS_LIB_MAP', None) is not None: lib_map.update(self.BLAS_LIB_MAP) - if hasattr(self, 'BLACS_LIB_MAP') and self.BLACS_LIB_MAP is not None: + if getattr(self, 'BLACS_LIB_MAP', None) is not None: lib_map.update(self.BLACS_LIB_MAP) - if hasattr(self, 'SCALAPACK_LIB_MAP') and self.SCALAPACK_LIB_MAP is not None: + if getattr(self, 'SCALAPACK_LIB_MAP', None) is not None: lib_map.update(self.SCALAPACK_LIB_MAP) self.SCALAPACK_LIB = self.variables.nappend('LIBSCALAPACK_ONLY', [x % lib_map for x in self.SCALAPACK_LIB]) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index a16fe35dc0..4c30faf19a 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -163,13 +163,13 @@ def _is_toolchain_for(cls, name): """see if this class can provide support for toolchain named name""" # TODO report later in the initialization the found version if name: - if hasattr(cls, 'NAME') and name == cls.NAME: - return True - else: + try: + return name == cls.NAME + except AttributeError: return False else: # is no name is supplied, check whether class can be used as a toolchain - return hasattr(cls, 'NAME') and cls.NAME + return bool(getattr(cls, 'NAME', None)) _is_toolchain_for = classmethod(_is_toolchain_for) @@ -330,10 +330,12 @@ def _copy_class_constants(self): if key not in self.CLASS_CONSTANT_COPIES: self.CLASS_CONSTANT_COPIES[key] = {} for cst in self.CLASS_CONSTANTS_TO_RESTORE: - if hasattr(self, cst): - self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(getattr(self, cst)) - else: + try: + value = getattr(self, cst) + except AttributeError: raise EasyBuildError("Class constant '%s' to be restored does not exist in %s", cst, self) + else: + self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(value) self.log.devel("Copied class constants: %s", self.CLASS_CONSTANT_COPIES[key]) @@ -342,10 +344,12 @@ def _restore_class_constants(self): key = self.__class__ for cst in self.CLASS_CONSTANT_COPIES[key]: newval = copy.deepcopy(self.CLASS_CONSTANT_COPIES[key][cst]) - if hasattr(self, cst): - self.log.devel("Restoring class constant '%s' to %s (was: %s)", cst, newval, getattr(self, cst)) - else: + try: + oldval = getattr(self, cst) + except AttributeError: self.log.devel("Restoring (currently undefined) class constant '%s' to %s", cst, newval) + else: + self.log.devel("Restoring class constant '%s' to %s (was: %s)", cst, newval, oldval) setattr(self, cst, newval) diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 48a586c567..56242b8fe4 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -63,7 +63,7 @@ def search_toolchain(name): package = easybuild.tools.toolchain check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX - if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name): + if not getattr(package, check_attr_name, None): # import all available toolchains, so we know about them tc_modules = import_available_modules('easybuild.toolchains') @@ -76,7 +76,7 @@ def search_toolchain(name): if hasattr(elem, '__module__'): # exclude the toolchain class defined in that module if not tc_mod.__file__ == sys.modules[elem.__module__].__file__: - elem_name = elem.__name__ if hasattr(elem, '__name__') else elem + elem_name = getattr(elem, '__name__', elem) _log.debug("Adding %s to list of imported classes used for looking for constants", elem_name) mod_classes.append(elem) @@ -89,13 +89,14 @@ def search_toolchain(name): tc_const_value = getattr(mod_class_mod, elem) _log.debug("Found constant %s ('%s') in module %s, adding it to %s", tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) - if hasattr(package, tc_const_name): + try: cur_value = getattr(package, tc_const_name) + except AttributeError: + setattr(package, tc_const_name, tc_const_value) + else: if not tc_const_value == cur_value: raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.", package.__name__, tc_const_name, cur_value, tc_const_value) - else: - setattr(package, tc_const_name, tc_const_value) # indicate that processing of toolchain constants is done, so it's not done again setattr(package, check_attr_name, True) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 1707d6ec54..45f51991ff 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -366,17 +366,14 @@ def setUp(self): self.orig_is_readable = st.is_readable self.orig_read_file = st.read_file self.orig_run_cmd = st.run_cmd - self.orig_platform_dist = st.platform.dist if hasattr(st.platform, 'dist') else None + self.orig_platform_dist = getattr(st.platform, 'dist', None) self.orig_platform_uname = st.platform.uname self.orig_get_tool_version = st.get_tool_version self.orig_sys_version_info = st.sys.version_info self.orig_HAVE_ARCHSPEC = st.HAVE_ARCHSPEC self.orig_HAVE_DISTRO = st.HAVE_DISTRO self.orig_ETC_OS_RELEASE = st.ETC_OS_RELEASE - if hasattr(st, 'archspec_cpu_host'): - self.orig_archspec_cpu_host = st.archspec_cpu_host - else: - self.orig_archspec_cpu_host = None + self.orig_archspec_cpu_host = getattr(st, 'archspec_cpu_host', None) def tearDown(self): """Cleanup after systemtools test.""" From 1fad79e11d714a66acd7afa747d461e48ce60399 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Mon, 30 Oct 2023 20:52:30 +0100 Subject: [PATCH 028/430] Adapt VERSION_REGEXP for EnvironmentModules When using an Environment Modules version built from git repository, version number contains git branch name, number of commit since last released version and commit hash. This commit adapts VERSION_REGEXP for EnvironmentModules class to allow using development or locally adapted versions of Environment Modules with EasyBuild. Fixes #4126 --- easybuild/tools/modules.py | 2 +- test/framework/modulestool.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 7d7181fe30..bdac20c274 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1326,7 +1326,7 @@ class EnvironmentModules(EnvironmentModulesTcl): COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' MAX_VERSION = None - VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' def __init__(self, *args, **kwargs): """Constructor, set Environment Modules-specific class variable values.""" diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index fb991ad797..f43a91e3b3 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -209,6 +209,21 @@ def test_environment_modules_specific(self): mt = EnvironmentModules(testing=True) self.assertIsInstance(mt.loaded_modules(), list) # dummy usage + # initialize Environment Modules tool with non-official version number + # pass (fake) full path to 'modulecmd.tcl' via $MODULES_CMD + fake_path = os.path.join(self.test_installpath, 'libexec', 'modulecmd.tcl') + fake_modulecmd_txt = '\n'.join([ + 'puts stderr {Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)}', + "puts {os.environ['FOO'] = 'foo'}", + ]) + write_file(fake_path, fake_modulecmd_txt) + os.chmod(fake_path, stat.S_IRUSR | stat.S_IXUSR) + os.environ['_module_raw'] = "() { eval `%s' bash $*`;\n}" % fake_path + os.environ['MODULES_CMD'] = fake_path + EnvironmentModules.COMMAND = fake_path + mt = EnvironmentModules(testing=True) + self.assertTrue(os.path.samefile(mt.cmd, fake_path), "%s - %s" % (mt.cmd, fake_path)) + def tearDown(self): """Testcase cleanup.""" super(ModulesToolTest, self).tearDown() From a3cb2ccdf6a8ead993d553e00a637ad747d1c5ff Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 13:23:23 +0100 Subject: [PATCH 029/430] Get available hidden modules on EnvironmentModules Environment Modules option "--all" was introduced in version 4.6.0 to obtain hidden modules among "module avail" results. Update EnvironmentModules class to make use of this option to return hidden modules on "available" function like done on Lmod class. Update "test_avail" unit test to match new results obtain with Environment Modules 4.6+ --- easybuild/tools/modules.py | 21 +++++++++++++++++++-- test/framework/modules.py | 5 +++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 7d7181fe30..d3c6019aba 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1283,14 +1283,14 @@ def tweak_stdout(txt): return super(EnvironmentModulesTcl, self).run_module(*args, **kwargs) - def available(self, mod_name=None): + def available(self, mod_name=None, extra_args=None): """ Return a list of available modules for the given (partial) module name; use None to obtain a list of all available modules. :param mod_name: a (partial) module name for filtering (default: None) """ - mods = super(EnvironmentModulesTcl, self).available(mod_name=mod_name) + mods = super(EnvironmentModulesTcl, self).available(mod_name=mod_name, extra_args=extra_args) # strip off slash at beginning, if it's there # under certain circumstances, 'modulecmd.tcl avail' (DEISA variant) spits out available modules like this clean_mods = [mod.lstrip(os.path.sep) for mod in mods] @@ -1328,6 +1328,8 @@ class EnvironmentModules(EnvironmentModulesTcl): MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + SHOW_HIDDEN_OPTION = '--all' + def __init__(self, *args, **kwargs): """Constructor, set Environment Modules-specific class variable values.""" # ensure in-depth modulepath search (MODULES_AVAIL_INDEPTH has been introduced in v4.3) @@ -1383,6 +1385,21 @@ def check_module_output(self, cmd, stdout, stderr): else: self.log.debug("No errors detected when running module command '%s'", cmd) + def available(self, mod_name=None, extra_args=None): + """ + Return a list of available modules for the given (partial) module name; + use None to obtain a list of all available modules. + + :param mod_name: a (partial) module name for filtering (default: None) + """ + if extra_args is None: + extra_args = [] + # make hidden modules visible (requires Environment Modules 4.6.0) + if StrictVersion(self.version) >= StrictVersion('4.6.0'): + extra_args.append(self.SHOW_HIDDEN_OPTION) + + return super(EnvironmentModules, self).available(mod_name=mod_name, extra_args=extra_args) + def get_setenv_value_from_modulefile(self, mod_name, var_name): """ Get value for specific 'setenv' statement from module file for the specified module. diff --git a/test/framework/modules.py b/test/framework/modules.py index c2460c7a11..9151527aac 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -218,6 +218,11 @@ def test_avail(self): self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) + elif (isinstance(self.modtool, EnvironmentModules) + and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')): + self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) + self.assertIn('toy/.0.0-deps', ms) + self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) else: self.assertEqual(len(ms), TEST_MODULES_COUNT) From fc27fe26a497b46fa19b7a565e001bd7187ac27a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 31 Dec 2023 15:37:13 +0100 Subject: [PATCH 030/430] use run_shell_cmd in scripts --- easybuild/scripts/findPythonDeps.py | 12 ++++++------ easybuild/scripts/mk_tmpl_easyblock_for.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index d6e496a048..31c990c468 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -48,7 +48,7 @@ def can_run(cmd, argument): return False -def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): +def run_shell_cmd(arguments, action_desc, capture_stderr=True, **kwargs): """Run the command and return the return code and output""" extra_args = kwargs or {} if sys.version_info[0] >= 3: @@ -66,7 +66,7 @@ def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): def run_in_venv(cmd, venv_path, action_desc): """Run the given command in the virtualenv at the given path""" cmd = 'source %s/bin/activate && %s' % (venv_path, cmd) - return run_cmd(cmd, action_desc, shell=True, executable='/bin/bash') + return run_shell_cmd(cmd, action_desc, shell=True, executable='/bin/bash') def get_dep_tree(package_spec, verbose): @@ -78,7 +78,7 @@ def get_dep_tree(package_spec, verbose): venv_dir = os.path.join(tmp_dir, 'venv') if verbose: print('Creating virtualenv at ' + venv_dir) - run_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') + run_shell_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') if verbose: print('Updating pip in virtualenv') run_in_venv('pip install --upgrade pip', venv_dir, action_desc='update pip') @@ -169,7 +169,7 @@ def print_deps(package, verbose): sys.exit(1) if args.verbose: print('Checking with EasyBuild for missing dependencies') - missing_dep_out = run_cmd(['eb', args.ec, '--missing'], + missing_dep_out = run_shell_cmd(['eb', args.ec, '--missing'], capture_stderr=False, action_desc='Get missing dependencies' ) @@ -189,7 +189,7 @@ def print_deps(package, verbose): os.chdir(tmp_dir) if args.verbose: print('Running EasyBuild to get build environment') - run_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment') + run_shell_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment') os.chdir(old_dir) cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package) @@ -197,7 +197,7 @@ def print_deps(package, verbose): cmd += ' --verbose' print('Restarting script in new build environment') - out = run_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') + out = run_shell_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') print(out) else: if not can_run('virtualenv', '--version'): diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index c5c7c0110a..1cd63393d6 100755 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -122,7 +122,7 @@ import easybuild.tools.toolchain as toolchain %(parent_import)s from easybuild.framework.easyconfig import CUSTOM, MANDATORY -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class %(class_name)s(%(parent)s): @@ -150,7 +150,7 @@ def configure_step(self): env.setvar('CUSTOM_ENV_VAR', 'foo') cmd = "configure command" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) # complete configuration with configure_method of parent super(%(class_name)s, self).configure_step() @@ -167,20 +167,20 @@ def build_step(self): # enable parallel build par = self.cfg['parallel'] cmd = "build command --parallel %%d --compiler-family %%s" %% (par, comp_fam) - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def test_step(self): \"\"\"Custom built-in test procedure for %(name)s.\"\"\" if self.cfg['runtest']: cmd = "test-command" - run_cmd(cmd, simple=True, log_all=True, log_ok=True) + run_shell_cmd(cmd) def install_step(self): \"\"\"Custom install procedure for %(name)s.\"\"\" cmd = "install command" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def sanity_check_step(self): \"\"\"Custom sanity check for %(name)s.\"\"\" From bc76fb3cb8562f5be58f82e3a346c45ad87352e0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 1 Jan 2024 19:02:03 +0100 Subject: [PATCH 031/430] fix code style in findPythonDeps.py script --- easybuild/scripts/findPythonDeps.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 31c990c468..1933bcf1df 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -170,9 +170,8 @@ def print_deps(package, verbose): if args.verbose: print('Checking with EasyBuild for missing dependencies') missing_dep_out = run_shell_cmd(['eb', args.ec, '--missing'], - capture_stderr=False, - action_desc='Get missing dependencies' - ) + capture_stderr=False, + action_desc='Get missing dependencies') excluded_dep = '(%s)' % os.path.basename(args.ec) missing_deps = [dep for dep in missing_dep_out.split('\n') if dep.startswith('*') and excluded_dep not in dep From 38f32bdebc4fa4a029b1d0d8650bfb86039f280c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Jan 2024 20:15:11 +0100 Subject: [PATCH 032/430] implement support for 'stream_output' option in run_shell_cmd --- easybuild/tools/run.py | 52 ++++++++++++++++++++++++++++++++---------- test/framework/run.py | 34 +++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 1a03cbe08e..0bb0d3c9f4 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -182,7 +182,7 @@ def cache_aware_func(cmd, *args, **kwargs): @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, - output_file=True, stream_output=False, asynchronous=False, with_hooks=True, + output_file=True, stream_output=None, asynchronous=False, with_hooks=True, qa_patterns=None, qa_wait_patterns=None): """ Run specified (interactive) shell command, and capture output + exit code. @@ -197,7 +197,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param work_dir: working directory to run command in (current working directory if None) :param use_bash: execute command through bash shell (enabled by default) :param output_file: collect command output in temporary output file - :param stream_output: stream command output to stdout + :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None) :param asynchronous: run command asynchronously :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers @@ -222,7 +222,7 @@ def to_cmd_str(cmd): return cmd_str # temporarily raise a NotImplementedError until all options are implemented - if any((stream_output, asynchronous)): + if asynchronous: raise NotImplementedError if qa_patterns or qa_wait_patterns: @@ -234,6 +234,11 @@ def to_cmd_str(cmd): cmd_str = to_cmd_str(cmd) cmd_name = os.path.basename(cmd_str.split(' ')[0]) + # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely + if stream_output is None and build_option('logtostdout'): + _log.info(f"Auto-enabling streaming output of '{cmd_str}' command because logging to stdout is enabled") + stream_output = True + # temporary output file(s) for command output if output_file: toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') @@ -264,9 +269,8 @@ def to_cmd_str(cmd): if not hidden: cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp) - if stdin: - # 'input' value fed to subprocess.run must be a byte sequence - stdin = stdin.encode() + if stream_output: + print_msg(f"(streaming) output for command '{cmd_str}':") # use bash as shell instead of the default /bin/sh used by subprocess.run # (which could be dash instead of bash, like on Ubuntu, see https://wiki.ubuntu.com/DashAsBinSh) @@ -276,8 +280,6 @@ def to_cmd_str(cmd): else: executable, shell = None, False - stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT - if with_hooks: hooks = load_hooks(build_option('hooks')) hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': work_dir}) @@ -286,13 +288,39 @@ def to_cmd_str(cmd): cmd_str = to_cmd_str(cmd) _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) + stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT + _log.info(f"Running command '{cmd_str}' in {work_dir}") - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=stderr, check=False, - cwd=work_dir, env=env, input=stdin, shell=shell, executable=executable) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, stdin=subprocess.PIPE, + cwd=work_dir, env=env, shell=shell, executable=executable) + + # 'input' value fed to subprocess.run must be a byte sequence + if stdin: + stdin = stdin.encode() + + if stream_output: + if stdin: + proc.stdin.write(stdin) + + exit_code = None + stdout, stderr = b'', b'' + + while exit_code is None: + exit_code = proc.poll() + + # use small read size (128 bytes) when streaming output, to make it stream more fluently + # -1 means reading until EOF + read_size = 128 if exit_code is None else -1 + + stdout += proc.stdout.read(read_size) + if split_stderr: + stderr += proc.stderr.read(read_size) + else: + (stdout, stderr) = proc.communicate(input=stdin) # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out) - output = proc.stdout.decode('utf-8', 'ignore') - stderr = proc.stderr.decode('utf-8', 'ignore') if split_stderr else None + output = stdout.decode('utf-8', 'ignore') + stderr = stderr.decode('utf-8', 'ignore') if split_stderr else None # store command output to temporary file(s) if output_file: diff --git a/test/framework/run.py b/test/framework/run.py index 4347dcdeeb..db74940aec 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1107,6 +1107,40 @@ def test_run_cmd_stream(self): for line in expected: self.assertIn(line, stdout) + def test_run_shell_cmd_stream(self): + """Test use of run_shell_cmd with streaming output.""" + self.mock_stdout(True) + self.mock_stderr(True) + cmd = '; '.join([ + "echo hello there", + "sleep 1", + "echo testing command that produces a fair amount of output", + "sleep 1", + "echo more than 128 bytes which means a whole bunch of characters...", + "sleep 1", + "echo more than 128 characters in fact, which is quite a bit when you think of it", + ]) + res = run_shell_cmd(cmd, stream_output=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + expected_output = '\n'.join([ + "hello there", + "testing command that produces a fair amount of output", + "more than 128 bytes which means a whole bunch of characters...", + "more than 128 characters in fact, which is quite a bit when you think of it", + '', + ]) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, expected_output) + + self.assertEqual(stderr, '') + expected = ("== (streaming) output for command 'echo hello" + '\n' + expected_output).split('\n') + for line in expected: + self.assertIn(line, stdout) + def test_run_cmd_async(self): """Test asynchronously running of a shell command via run_cmd + complete_cmd.""" From c98825dbdfa874179ec1c55b210cc80105793983 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Jan 2024 20:15:19 +0100 Subject: [PATCH 033/430] disable stream_output when running shell commands in systemtools --- easybuild/tools/systemtools.py | 39 +++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 32caf0b4a0..a2dfb08185 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -273,7 +273,8 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity())) else: # BSD-type systems - res = run_shell_cmd('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) try: if int(res.output) > 0: core_cnt = int(res.output) @@ -310,7 +311,7 @@ def get_total_memory(): elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) if res.exit_code == 0: memtotal = int(res.output.strip()) // (1024**2) @@ -392,7 +393,8 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) out = res.output.strip() if res.exit_code == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] @@ -400,7 +402,7 @@ def get_cpu_vendor(): else: cmd = "sysctl -n machdep.cpu.brand_string" res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) out = res.output.strip().split(' ')[0] if res.exit_code == 0 and out in CPU_VENDORS: vendor = out @@ -503,7 +505,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) if res.exit_code == 0: model = res.output.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -548,7 +550,7 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) out = res.output.strip() cpu_freq = None if res.exit_code == 0 and out: @@ -597,7 +599,7 @@ def get_cpu_features(): cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) if res.exit_code == 0: cpu_feat.extend(res.output.strip().lower().split()) @@ -625,7 +627,7 @@ def get_gpu_info(): cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) if res.exit_code == 0: for line in res.output.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) @@ -644,14 +646,14 @@ def get_gpu_info(): cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) if res.exit_code == 0: amd_driver = res.output.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) if res.exit_code == 0: for line in res.output.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] @@ -870,7 +872,8 @@ def check_os_dependency(dep): pkg_cmd_flag.get(pkg_cmd), dep, ]) - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, + output_file=False, stream_output=False) found = res.exit_code == 0 if found: break @@ -882,7 +885,8 @@ def check_os_dependency(dep): # try locate if it's available if not found and which('locate'): cmd = 'locate -c --regexp "/%s$"' % dep - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, + output_file=False, stream_output=False) try: found = (res.exit_code == 0 and int(res.output.strip()) > 0) except ValueError: @@ -898,7 +902,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): Output is returned as a single-line string (newlines are replaced by '; '). """ res = run_shell_cmd(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, - hidden=True, with_hooks=False, output_file=False) + hidden=True, with_hooks=False, output_file=False, stream_output=False) if not ignore_ec and res.exit_code: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output)) return UNKNOWN @@ -910,7 +914,8 @@ def get_gcc_version(): """ Process `gcc --version` and return the GCC version. """ - res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True, output_file=False) + res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True, + output_file=False, stream_output=False) gcc_ver = None if res.exit_code: _log.warning("Failed to determine the version of GCC: %s", res.output) @@ -966,7 +971,7 @@ def get_linked_libs_raw(path): or None for other types of files. """ - res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False) + res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False, stream_output=False) if res.exit_code: fail_msg = "Failed to run 'file %s': %s" % (path, res.output) _log.warning(fail_msg) @@ -1001,7 +1006,7 @@ def get_linked_libs_raw(path): # take into account that 'ldd' may fail for strange reasons, # like printing 'not a dynamic executable' when not enough memory is available # (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111) - res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True, output_file=False) + res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True, output_file=False, stream_output=False) if res.exit_code == 0: linked_libs_out = res.output else: @@ -1183,7 +1188,7 @@ def get_default_parallelism(): # No cache -> Calculate value from current system values par = get_avail_core_count() # determine max user processes via ulimit -u - res = run_shell_cmd("ulimit -u", in_dry_run=True, hidden=True, output_file=False) + res = run_shell_cmd("ulimit -u", in_dry_run=True, hidden=True, output_file=False, stream_output=False) try: if res.output.startswith("unlimited"): maxuserproc = 2 ** 32 - 1 From 2d5db91d92c97ae3ee7ed22a50cb20f3eb56e0c6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 09:04:59 +0100 Subject: [PATCH 034/430] add comment to explain why hidden bzip2 module is not included in "avail" output for Tcl-based environment modules tool --- test/framework/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/modules.py b/test/framework/modules.py index 9151527aac..2a05395b7c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -220,6 +220,7 @@ def test_avail(self): self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')): + # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) From 7f1f7eac5dbcbe32f1bffaa4839bea28b3158a7e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 12:14:14 +0100 Subject: [PATCH 035/430] switch to run_shell_cmd in easybuild/tools/container --- easybuild/tools/containers/apptainer.py | 18 ++++++++-------- easybuild/tools/containers/docker.py | 4 ++-- easybuild/tools/containers/singularity.py | 18 ++++++++-------- easybuild/tools/containers/utils.py | 25 +++++++++++++---------- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py index 67a6db5bc3..8a294ce1ad 100644 --- a/easybuild/tools/containers/apptainer.py +++ b/easybuild/tools/containers/apptainer.py @@ -35,7 +35,7 @@ from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS from easybuild.tools.config import build_option, container_path from easybuild.tools.filetools import remove_file, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class ApptainerContainer(SingularityContainer): @@ -48,15 +48,15 @@ class ApptainerContainer(SingularityContainer): def apptainer_version(): """Get Apptainer version.""" version_cmd = "apptainer --version" - out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) - if ec: - raise EasyBuildError("Error running '%s': %s for tool {1} with output: {2}" % (version_cmd, out)) + res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) + if res.exit_code: + raise EasyBuildError(f"Error running '{version_cmd}': {res.output}") - res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) - if not res: - raise EasyBuildError("Error parsing Apptainer version: %s" % out) + regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) + if not regex_res: + raise EasyBuildError(f"Error parsing Apptainer version: {res.output}") - return res.group(0) + return regex_res.group(0) def build_image(self, recipe_path): """Build container image by calling out to 'sudo apptainer build'.""" @@ -108,5 +108,5 @@ def build_image(self, recipe_path): cmd = ' '.join(['sudo', cmd_env, apptainer, 'build', cmd_opts, img_path, recipe_path]) print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd) - run_cmd(cmd, stream_output=True) + run_shell_cmd(cmd, stream_output=True) print_msg("Apptainer image created at %s" % img_path, log=self.log) diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index aa37a90873..5a1597d634 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -36,7 +36,7 @@ from easybuild.tools.containers.utils import det_os_deps from easybuild.tools.filetools import remove_dir from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd DOCKER_TMPL_HEADER = """\ @@ -164,7 +164,7 @@ def build_image(self, dockerfile): docker_cmd = ' '.join(['sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.']) print_msg("Running '%s', you may need to enter your 'sudo' password..." % docker_cmd) - run_cmd(docker_cmd, path=tempdir, stream_output=True) + run_shell_cmd(docker_cmd, work_dir=tempdir, stream_output=True) print_msg("Docker image created at %s" % container_name, log=self.log) remove_dir(tempdir) diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index 41ec9829c7..638a985ae5 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -40,7 +40,7 @@ from easybuild.tools.config import build_option, container_path from easybuild.tools.containers.base import ContainerGenerator from easybuild.tools.filetools import read_file, remove_file, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd ARCH = 'arch' # Arch Linux @@ -162,15 +162,15 @@ class SingularityContainer(ContainerGenerator): def singularity_version(): """Get Singularity version.""" version_cmd = "singularity --version" - out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) - if ec: - raise EasyBuildError("Error running '%s': %s for tool {1} with output: {2}" % (version_cmd, out)) + res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) + if res.exit_code: + raise EasyBuildError(f"Error running '{version_cmd}': {res.output}") - res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) - if not res: - raise EasyBuildError("Error parsing Singularity version: %s" % out) + regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) + if not regex_res: + raise EasyBuildError(f"Error parsing Singularity version: {res.output}") - return res.group(0) + return regex_res.group(0) def resolve_template(self): """Return template container recipe.""" @@ -403,5 +403,5 @@ def build_image(self, recipe_path): cmd = ' '.join(['sudo', cmd_env, singularity, 'build', cmd_opts, img_path, recipe_path]) print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd) - run_cmd(cmd, stream_output=True) + run_shell_cmd(cmd, stream_output=True) print_msg("Singularity image created at %s" % img_path, log=self.log) diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py index e01a117427..d543ca77fc 100644 --- a/easybuild/tools/containers/utils.py +++ b/easybuild/tools/containers/utils.py @@ -36,7 +36,7 @@ from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.filetools import which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd def det_os_deps(easyconfigs): @@ -70,20 +70,23 @@ def check_tool(tool_name, min_tool_version=None): if not tool_path: return False - print_msg("{0} tool found at {1}".format(tool_name, tool_path)) + print_msg(f"{tool_name} tool found at {tool_path}") if not min_tool_version: return True - version_cmd = "{0} --version".format(tool_name) - out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) - if ec: - raise EasyBuildError("Error running '{0}' for tool {1} with output: {2}".format(version_cmd, tool_name, out)) - res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) - if not res: - raise EasyBuildError("Error parsing version for tool {0}".format(tool_name)) - tool_version = res.group(0) + version_cmd = f"{tool_name} --version" + res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) + if res.exit_code: + raise EasyBuildError(f"Error running '{version_cmd}' for tool {tool_name} with output: {res.output}") + + regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) + if not regex_res: + raise EasyBuildError(f"Error parsing version for tool {tool_name}") + + tool_version = regex_res.group(0) version_ok = LooseVersion(str(min_tool_version)) <= LooseVersion(tool_version) if version_ok: - print_msg("{0} version '{1}' is {2} or higher ... OK".format(tool_name, tool_version, min_tool_version)) + print_msg(f"{tool_name} version '{tool_version}' is {min_tool_version} or higher ... OK") + return version_ok From b05b07ddddfb43861015ab2eb685d73260ed8272 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:16:58 +0000 Subject: [PATCH 036/430] bump minimum required Lmod to 8.0.0 --- easybuild/tools/modules.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 36368fe6e0..7e106ae1d2 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1406,9 +1406,8 @@ class Lmod(ModulesTool): NAME = "Lmod" COMMAND = 'lmod' COMMAND_ENVIRONMENT = 'LMOD_CMD' - REQ_VERSION = '6.5.1' - DEPR_VERSION = '7.0.0' - REQ_VERSION_DEPENDS_ON = '7.6.1' + REQ_VERSION = '8.0.0' + DEPR_VERSION = '8.0.0' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" SHOW_HIDDEN_OPTION = '--show-hidden' @@ -1427,7 +1426,7 @@ def __init__(self, *args, **kwargs): super(Lmod, self).__init__(*args, **kwargs) version = StrictVersion(self.version) - self.supports_depends_on = version >= self.REQ_VERSION_DEPENDS_ON + self.supports_depends_on = True # See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html if version >= '8.7.12': self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod') @@ -1586,13 +1585,9 @@ def module_wrapper_exists(self, mod_name): Determine whether a module wrapper with specified name exists. First check for wrapper defined in .modulerc.lua, fall back to also checking .modulerc (Tcl syntax). """ - res = None - - # first consider .modulerc.lua with Lmod 7.8 (or newer) - if StrictVersion(self.version) >= StrictVersion('7.8'): - mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$' - res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua', - mod_wrapper_regex_template=mod_wrapper_regex_template) + mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$' + res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua', + mod_wrapper_regex_template=mod_wrapper_regex_template) # fall back to checking for .modulerc in Tcl syntax if res is None: From 21f1d5862166074dcd05d76a21f1b06c34f7022e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 15:20:24 +0100 Subject: [PATCH 037/430] fix log.deprecated statements for renamed methods for installing extensions --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/extension.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 914e8ad4a0..66a67e7b32 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1854,7 +1854,7 @@ def install_extensions(self, *args, **kwargs): """[DEPRECATED] Install extensions.""" self.log.deprecated( "Easyblock.install_extensions() is deprecated, use Easyblock.install_all_extensions() instead.", - '5.0', + '6.0', ) self.install_all_extensions(*args, **kwargs) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 31cfb8d373..1aa53656e7 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -177,7 +177,7 @@ def prerun(self): """ self.log.deprecated( "Extension.prerun() is deprecated, use Extension.pre_install_extension() instead.", - '5.0', + '6.0', ) self.pre_install_extension() @@ -193,7 +193,7 @@ def run(self, *args, **kwargs): """ self.log.deprecated( "Extension.run() is deprecated, use Extension.install_extension() instead.", - '5.0', + '6.0', ) self.install_extension(*args, **kwargs) @@ -209,7 +209,7 @@ def run_async(self, *args, **kwargs): """ self.log.deprecated( "Extension.run_async() is deprecated, use Extension.install_extension_async() instead.", - '5.0', + '6.0', ) self.install_extension_async(*args, **kwargs) @@ -225,7 +225,7 @@ def postrun(self): """ self.log.deprecated( "Extension.postrun() is deprecated, use Extension.post_install_extension() instead.", - '5.0', + '6.0', ) self.post_install_extension() From 5cdea4615623479f68d2f34169a971c5cef26e25 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:35:39 +0000 Subject: [PATCH 038/430] remove Lmod 7 testing --- .github/workflows/unit_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 08408aea75..225b3f85c6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -13,7 +13,6 @@ jobs: setup: runs-on: ubuntu-20.04 outputs: - lmod7: Lmod-7.8.22 lmod8: Lmod-8.7.6 modulesTcl: modules-tcl-1.147 modules3: modules-3.2.10 @@ -29,7 +28,6 @@ jobs: modules_tool: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context - - ${{needs.setup.outputs.lmod7}} - ${{needs.setup.outputs.lmod8}} - ${{needs.setup.outputs.modulesTcl}} - ${{needs.setup.outputs.modules3}} From d7f2fafcfca7df863b58a1ef1ad4421138d5a557 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 16:03:53 +0100 Subject: [PATCH 039/430] tweak warning/error message when .mod files are found --- easybuild/framework/easyblock.py | 6 ++++-- test/framework/toy_build.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 954bb3212e..4fba08ccab 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3312,7 +3312,7 @@ def sanity_check_mod_files(self): fail_msg = None if mod_files: - fail_msg = ".mod files (%s) found in the installation." % ', '.join(mod_files) + fail_msg = f"One or more .mod files found in {self.installdir}: " + ', '.join(mod_files) return fail_msg @@ -3638,7 +3638,9 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_sanity_check']: + # software installed with GCCcore toolchain should not have Fortran module files (.mod), + # unless that's explicitly allowed + if self.toolchain.name in ('GCCcore',) and not self.cfg['skip_mod_files_sanity_check']: mod_files_found_msg = self.sanity_check_mod_files() if mod_files_found_msg: if build_option('fail_on_mod_files_gcccore'): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9bf85607fa..909c3b6664 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3979,11 +3979,11 @@ def test_toy_mod_files(self): stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) - pattern = r"WARNING: .mod files (.*) found in the installation." + pattern = r"WARNING: One or more \.mod files found in .*/software/toy/0.0-GCCcore-6.2.0: .*/lib64/file.mod" self.assertRegex(stderr.strip(), pattern) args += ['--fail-on-mod-files-gcccore'] - pattern = r"Sanity check failed: .mod files (.*) found in the installation." + pattern = r"Sanity check failed: One or more \.mod files found in .*/toy/0.0-GCCcore-6.2.0: .*/lib/file.mod" self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, extra_args=args, verify=False, fails=True, verbose=False, raise_error=True) From 8b4a889a2acc8bbc0be04d018da2e1e1c9ae2265 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 21:02:05 +0100 Subject: [PATCH 040/430] switch to run_shell_cmd in toy_build test module --- .../easybuild/easyblocks/t/toy_buggy.py | 2 +- test/framework/toy_build.py | 58 ++++++++++--------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py index 3695744630..43d7433265 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py @@ -41,7 +41,7 @@ def configure_step(self): def build_step(self): """Build toy.""" # note: import is (purposely) missing, so this will go down hard - run_cmd('gcc toy.c -o toy') # noqa + run_shell_cmd('gcc toy.c -o toy') # noqa def install_step(self): """Install toy.""" diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 909c3b6664..d4c7988534 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -58,7 +58,7 @@ from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import nub from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -374,7 +374,7 @@ def test_toy_buggy_easyblock(self): 'verify': False, 'verbose': False, } - err_regex = r"name 'run_cmd' is not defined" + err_regex = r"name 'run_shell_cmd' is not defined" self.assertErrorRegex(NameError, err_regex, self.run_test_toy_build_with_output, **kwargs) def test_toy_build_formatv2(self): @@ -742,9 +742,9 @@ def test_toy_group_check(self): # figure out a group that we're a member of to use in the test with self.mocked_stdout_stderr(): - out, ec = run_cmd('groups', simple=False) - self.assertEqual(ec, 0, "Failed to select group to use in test") - group_name = out.split(' ')[0].strip() + res = run_shell_cmd('groups') + self.assertEqual(res.exit_code, 0, "Failed to select group to use in test") + group_name = res.output.split(' ')[0].strip() toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -1284,8 +1284,8 @@ def test_toy_extension_patches_postinstallcmds(self): # make sure that patches were actually applied (without them the message producded by 'bar' is different) bar_bin = os.path.join(installdir, 'bin', 'bar') with self.mocked_stdout_stderr(): - out, _ = run_cmd(bar_bin) - self.assertEqual(out, "I'm a bar, and very very proud of it.\n") + res = run_shell_cmd(bar_bin) + self.assertEqual(res.output, "I'm a bar, and very very proud of it.\n") # verify that post-install command for 'bar' extension was executed fn = 'created-via-postinstallcmds.txt' @@ -2819,17 +2819,19 @@ def test_toy_filter_rpath_sanity_libs(self): libtoy_libdir = os.path.join(self.test_installpath, 'software', 'libtoy', '0.0', 'lib') toyapp_bin = os.path.join(self.test_installpath, 'software', 'toy-app', '0.0', 'bin', 'toy-app') - rpath_regex = re.compile(r"RPATH.*%s" % libtoy_libdir, re.M) + rpath_regex = re.compile(r"RPATH.*" + libtoy_libdir, re.M) with self.mocked_stdout_stderr(): - out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False) - self.assertTrue(rpath_regex.search(out), "Pattern '%s' should be found in: %s" % (rpath_regex.pattern, out)) + res = run_shell_cmd(f"readelf -d {toyapp_bin}") + self.assertTrue(rpath_regex.search(res.output), + f"Pattern '{rpath_regex.pattern}' should be found in: {res.output}") with self.mocked_stdout_stderr(): - out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False) + res = run_shell_cmd(f"ldd {toyapp_bin}") + out = res.output libtoy_regex = re.compile(r"libtoy.so => /.*/libtoy.so", re.M) notfound = re.compile(r"libtoy\.so\s*=>\s*not found", re.M) - self.assertTrue(libtoy_regex.search(out), "Pattern '%s' should be found in: %s" % (libtoy_regex.pattern, out)) - self.assertFalse(notfound.search(out), "Pattern '%s' should not be found in: %s" % (notfound.pattern, out)) + self.assertTrue(libtoy_regex.search(out), f"Pattern '{libtoy_regex.pattern}' should be found in: {out}") + self.assertFalse(notfound.search(out), f"Pattern '{notfound.pattern}' should not be found in: {out}") # test sanity error when --rpath-filter is used to filter a required library # In this test, libtoy.so will be linked, but not RPATH-ed due to the --rpath-filter @@ -2848,16 +2850,16 @@ def test_toy_filter_rpath_sanity_libs(self): self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True) with self.mocked_stdout_stderr(): - out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False) - self.assertFalse(rpath_regex.search(out), - "Pattern '%s' should not be found in: %s" % (rpath_regex.pattern, out)) + res = run_shell_cmd(f"readelf -d {toyapp_bin}") + self.assertFalse(rpath_regex.search(res.output), + f"Pattern '{rpath_regex.pattern}' should not be found in: {res.output}") with self.mocked_stdout_stderr(): - out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False) - self.assertFalse(libtoy_regex.search(out), - "Pattern '%s' should not be found in: %s" % (libtoy_regex.pattern, out)) - self.assertTrue(notfound.search(out), - "Pattern '%s' should be found in: %s" % (notfound.pattern, out)) + res = run_shell_cmd(f"ldd {toyapp_bin}") + self.assertFalse(libtoy_regex.search(res.output), + f"Pattern '{libtoy_regex.pattern}' should not be found in: {res.output}") + self.assertTrue(notfound.search(res.output), + f"Pattern '{notfound.pattern}' should be found in: {res.output}") # test again with list of library names passed to --filter-rpath-sanity-libs args = ['--rpath', '--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libfoo.so,libtoy.so,libbar.so'] @@ -2865,16 +2867,16 @@ def test_toy_filter_rpath_sanity_libs(self): self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True) with self.mocked_stdout_stderr(): - out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False) + res = run_shell_cmd(f"readelf -d {toyapp_bin}") self.assertFalse(rpath_regex.search(out), - "Pattern '%s' should not be found in: %s" % (rpath_regex.pattern, out)) + f"Pattern '{rpath_regex.pattern}' should not be found in: {res.output}") with self.mocked_stdout_stderr(): - out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False) - self.assertFalse(libtoy_regex.search(out), - "Pattern '%s' should not be found in: %s" % (libtoy_regex.pattern, out)) - self.assertTrue(notfound.search(out), - "Pattern '%s' should be found in: %s" % (notfound.pattern, out)) + res = run_shell_cmd(f"ldd {toyapp_bin}") + self.assertFalse(libtoy_regex.search(res.output), + f"Pattern '{libtoy_regex.pattern}' should not be found in: {res.output}") + self.assertTrue(notfound.search(res.output), + f"Pattern '{notfound.pattern}' should be found in: {res.output}") def test_toy_modaltsoftname(self): """Build two dependent toys as in test_toy_toy but using modaltsoftname""" From 84b8e5d71de59e1198a4dfb654f8cf2ef83a13df Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 21:55:42 +0100 Subject: [PATCH 041/430] switch to run_shell_cmd in toolchain test module --- test/framework/toolchain.py | 133 ++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 66 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index aa090bda94..1c8c1771a6 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -50,7 +50,7 @@ from easybuild.tools.environment import setvar from easybuild.tools.filetools import adjust_permissions, copy_dir, find_eb_script, mkdir from easybuild.tools.filetools import read_file, symlink, write_file, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template from easybuild.tools.toolchain.toolchain import env_vars_external_module @@ -2366,8 +2366,8 @@ def test_rpath_args_script(self): # simplest possible compiler command with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s' -c foo.c" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' -c foo.c") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2378,12 +2378,12 @@ def test_rpath_args_script(self): "'-c'", "'foo.c'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # linker command, --enable-new-dtags should be replaced with --disable-new-dtags with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s ld '' '%s' --enable-new-dtags foo.o" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} ld '' '{rpath_inc}' --enable-new-dtags foo.o") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2394,12 +2394,12 @@ def test_rpath_args_script(self): "'--disable-new-dtags'", "'foo.o'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # compiler command, -Wl,--enable-new-dtags should be replaced with -Wl,--disable-new-dtags with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s' -Wl,--enable-new-dtags foo.c" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' -Wl,--enable-new-dtags foo.c") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2410,12 +2410,12 @@ def test_rpath_args_script(self): "'-Wl,--disable-new-dtags'", "'foo.c'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing no arguments with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s'" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}'") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2424,12 +2424,12 @@ def test_rpath_args_script(self): "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing a single empty argument with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s ld.gold '' '%s' ''" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} ld.gold '' '{rpath_inc}' ''") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2439,13 +2439,13 @@ def test_rpath_args_script(self): "'--disable-new-dtags'", "''", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument, but non-existing path => not used in RPATH, but -L option is retained - cmd = "%s gcc '' '%s' foo.c -L%s/foo -lfoo" % (script, rpath_inc, self.test_prefix) + cmd = f"{script} gcc '' '{rpath_inc}' foo.c -L{self.test_prefix}/foo -lfoo" with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2457,13 +2457,13 @@ def test_rpath_args_script(self): "'-L%s/foo'" % self.test_prefix, "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument again, with existing path mkdir(os.path.join(self.test_prefix, 'foo')) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2476,12 +2476,12 @@ def test_rpath_args_script(self): "'-L%s/foo'" % self.test_prefix, "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # relative paths passed to -L are *not* RPATH'ed in with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s' foo.c -L../lib -lfoo" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' foo.c -L../lib -lfoo") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2493,13 +2493,13 @@ def test_rpath_args_script(self): "'-L../lib'", "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument, with value separated by a space - cmd = "%s gcc '' '%s' foo.c -L %s/foo -lfoo" % (script, rpath_inc, self.test_prefix) + cmd = f"{script} gcc '' '{rpath_inc}' foo.c -L {self.test_prefix}/foo -lfoo" with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2512,7 +2512,7 @@ def test_rpath_args_script(self): "'-L%s/foo'" % self.test_prefix, "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) mkdir(os.path.join(self.test_prefix, 'bar')) mkdir(os.path.join(self.test_prefix, 'lib64')) @@ -2534,8 +2534,8 @@ def test_rpath_args_script(self): '-L%s/bar' % self.test_prefix, ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2556,7 +2556,7 @@ def test_rpath_args_script(self): "'-L/usr/lib'", "'-L%s/bar'" % self.test_prefix, ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test specifying of custom rpath filter cmd = ' '.join([ @@ -2572,8 +2572,8 @@ def test_rpath_args_script(self): '-lbar', ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2589,7 +2589,7 @@ def test_rpath_args_script(self): "'-L/bar'", "'-lbar'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # slightly trimmed down real-life example (compilation of XZ) for subdir in ['icc/lib/intel64', 'imkl/lib', 'imkl/mkl/lib/intel64', 'gettext/lib']: @@ -2612,8 +2612,8 @@ def test_rpath_args_script(self): '-Wl,/example/software/XZ/5.2.2-intel-2016b/lib', ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s icc '' '%s' %s" % (script, rpath_inc, args), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} icc '' '{rpath_inc}' {args}") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2640,7 +2640,7 @@ def test_rpath_args_script(self): "'-Wl,-rpath'", "'-Wl,/example/software/XZ/5.2.2-intel-2016b/lib'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # trimmed down real-life example involving quotes and escaped quotes (compilation of GCC) args = [ @@ -2657,8 +2657,8 @@ def test_rpath_args_script(self): ] cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, ' '.join(args)) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -2678,15 +2678,16 @@ def test_rpath_args_script(self): "'-o' 'build/version.o'", "'../../gcc/version.c'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # verify that no -rpath arguments are injected when command is run in 'version check' mode for extra_args in ["-v", "-V", "--version", "-dumpversion", "-v -L/test/lib"]: cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, extra_args) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(["'%s'" % x for x in extra_args.split(' ')])) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) + cmd_args = ' '.join(["'%s'" % x for x in extra_args.split(' ')]) + self.assertEqual(res.output.strip(), f"CMD_ARGS=({cmd_args})") # if a compiler command includes "-x c++-header" or "-x c-header" (which imply no linking is done), # we should *not* inject -Wl,-rpath options, since those enable linking as a side-effect; @@ -2699,10 +2700,10 @@ def test_rpath_args_script(self): for extra_args in test_cases: cmd = "%s g++ '' '%s' foo.c -O2 %s" % (script, rpath_inc, extra_args) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = ["'foo.c'", "'-O2'"] + ["'%s'" % x for x in extra_args.split(' ')] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # check whether $LIBRARY_PATH is taken into account test_cmd_gcc = "%s gcc '' '%s' -c foo.c" % (script, rpath_inc) @@ -2768,16 +2769,16 @@ def test_rpath_args_script(self): os.environ['LIBRARY_PATH'] = ':'.join(library_path) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_gcc, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_gcc) + self.assertEqual(res.exit_code, 0) cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path if x] + post_cmd_args_gcc - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_ld, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_ld) + self.assertEqual(res.exit_code, 0) cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % x for x in library_path if x] + post_cmd_args_ld - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # paths already listed via -L don't get included again as RPATH option new_lib64 = os.path.join(self.test_prefix, 'new', 'lib64') @@ -2796,19 +2797,19 @@ def test_rpath_args_script(self): os.environ['LIBRARY_PATH'] = ':'.join(library_path) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_gcc, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_gcc) + self.assertEqual(res.exit_code, 0) # no -L options in GCC command, so all $LIBRARY_PATH entries are retained except for last one (lib symlink) cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path[:-1] if x] + post_cmd_args_gcc - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_ld, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_ld) + self.assertEqual(res.exit_code, 0) # only new path from $LIBRARY_PATH is included as -rpath option, # since others are already included via corresponding -L flag cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % new_lib64] + post_cmd_args_ld - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath""" @@ -2926,8 +2927,8 @@ def test_toolchain_prepare_rpath(self): '-DX="\\"\\""', ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) expected = ' '.join([ '-Wl,--disable-new-dtags', '-Wl,-rpath=%s/foo' % self.test_prefix, @@ -2937,7 +2938,7 @@ def test_toolchain_prepare_rpath(self): '$FOO', '-DX=""', ]) - self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')}) + self.assertEqual(res.output.strip(), expected % {'user': os.getenv('USER')}) # check whether 'stubs' library directory are correctly filtered out paths = [ @@ -2963,8 +2964,8 @@ def test_toolchain_prepare_rpath(self): cmd = "g++ ${USER}.c %s" % ' '.join(args) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) expected = ' '.join([ '-Wl,--disable-new-dtags', @@ -2989,7 +2990,7 @@ def test_toolchain_prepare_rpath(self): '-L%s/prefix/software/bleh/0/lib/stubs' % self.test_prefix, '-L%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix, ]) - self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')}) + self.assertEqual(res.output.strip(), expected % {'user': os.getenv('USER')}) # calling prepare() again should *not* result in wrapping the existing RPATH wrappers # this can happen when building extensions From 47f7a20941becaa44397674cba99b4a7f1dc8617 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 13:25:33 +0100 Subject: [PATCH 042/430] switch to run_shell_cmd where possible in easybuild.* modules --- easybuild/framework/extension.py | 8 ++++---- easybuild/toolchains/linalg/flexiblas.py | 6 +++--- easybuild/tools/job/slurm.py | 18 +++++++++--------- easybuild/tools/options.py | 7 ++++--- easybuild/tools/package/utilities.py | 4 ++-- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 8dc4669392..d30242495c 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -42,7 +42,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import check_async_cmd, run_cmd +from easybuild.tools.run import check_async_cmd, run_cmd, run_shell_cmd def resolve_exts_filter_template(exts_filter, ext): @@ -274,14 +274,14 @@ def sanity_check_step(self): elif exts_filter: cmd, stdin = resolve_exts_filter_template(exts_filter, self) # set log_ok to False so we can catch the error instead of run_cmd - (output, ec) = run_cmd(cmd, log_ok=False, simple=False, regexp=False, inp=stdin) + cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) - if ec: + if cmd_res.exit_code: if stdin: fail_msg = 'command "%s" (stdin: "%s") failed' % (cmd, stdin) else: fail_msg = 'command "%s" failed' % cmd - fail_msg += "; output:\n%s" % output.strip() + fail_msg += "; output:\n%s" % cmd_res.output.strip() self.log.warning("Sanity check for '%s' extension failed: %s", self.name, fail_msg) res = (False, fail_msg) # keep track of all reasons of failure diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index c266aff248..3726e7444f 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -34,7 +34,7 @@ from easybuild.tools.toolchain.linalg import LinAlg -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -48,11 +48,11 @@ def det_flexiblas_backend_libs(): # System-wide (config directory): # OPENBLAS # library = libflexiblas_openblas.so - out, _ = run_cmd("flexiblas list", simple=False, trace=False) + res = run_shell_cmd("flexiblas list", hidden=True) shlib_ext = get_shared_lib_ext() flexiblas_lib_regex = re.compile(r'library = (?Plib.*\.%s)' % shlib_ext, re.M) - flexiblas_libs = flexiblas_lib_regex.findall(out) + flexiblas_libs = flexiblas_lib_regex.findall(res.output) backend_libs = [] for flexiblas_lib in flexiblas_libs: diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py index 1f2dea776d..97a925bd35 100644 --- a/easybuild/tools/job/slurm.py +++ b/easybuild/tools/job/slurm.py @@ -37,7 +37,7 @@ from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option from easybuild.tools.job.backend import JobBackend from easybuild.tools.filetools import which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd _log = fancylogger.getLogger('slurm', fname=False) @@ -78,8 +78,8 @@ def __init__(self, *args, **kwargs): def _check_version(self): """Check whether version of Slurm complies with required version.""" - (out, _) = run_cmd("sbatch --version", trace=False) - slurm_ver = out.strip().split(' ')[-1] + res = run_shell_cmd("sbatch --version", hidden=True) + slurm_ver = res.output.strip().split(' ')[-1] self.log.info("Found Slurm version %s", slurm_ver) if LooseVersion(slurm_ver) < LooseVersion(self.REQ_VERSION): @@ -116,16 +116,16 @@ def queue(self, job, dependencies=frozenset()): else: submit_cmd += ' --%s "%s"' % (key, job.job_specs[key]) - (out, _) = run_cmd(submit_cmd, trace=False) + cmd_res = run_shell_cmd(submit_cmd, hidden=True) jobid_regex = re.compile("^Submitted batch job (?P[0-9]+)") - res = jobid_regex.search(out) - if res: - job.jobid = res.group('jobid') + regex_res = jobid_regex.search(cmd_res.output) + if regex_res: + job.jobid = regex_res.group('jobid') self.log.info("Job submitted, got job ID %s", job.jobid) else: - raise EasyBuildError("Failed to determine job ID from output of submission command: %s", out) + raise EasyBuildError("Failed to determine job ID from output of submission command: %s", cmd_res.output) self._submitted.append(job) @@ -142,7 +142,7 @@ def complete(self): job_ids.append(job.jobid) if job_ids: - run_cmd("scontrol release %s" % ' '.join(job_ids), trace=False) + run_shell_cmd("scontrol release %s" % ' '.join(job_ids), hidden=True) submitted_jobs = '; '.join(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in self._submitted]) print_msg("List of submitted jobs (%d): %s" % (len(self._submitted), submitted_jobs), log=self.log) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 0d2e5eb853..be5cf94f75 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -98,7 +98,7 @@ from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes from easybuild.tools.modules import Lmod from easybuild.tools.robot import det_robot_path -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME @@ -1893,8 +1893,9 @@ def set_tmpdir(tmpdir=None, raise_error=False): fd, tmptest_file = tempfile.mkstemp() os.close(fd) os.chmod(tmptest_file, 0o700) - if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, - stream_output=False, with_hooks=False): + res = run_shell_cmd(tmptest_file, fail_on_error=False, in_dry_run=True, hidden=True, stream_output=False, + with_hooks=False) + if res.exit_code: msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() msg += "This can cause problems in the build process, consider using --tmpdir." if raise_error: diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 9a695e33da..adaa0814fc 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -45,7 +45,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, which from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import get_subclasses, import_available_modules @@ -145,7 +145,7 @@ def package_with_fpm(easyblock): ]) cmd = ' '.join(cmdlist) _log.debug("The flattened cmdlist looks like: %s", cmd) - run_cmd(cmdlist, log_all=True, simple=True, shell=False) + run_shell_cmd(cmdlist, use_bash=False) _log.info("Created %s package(s) in %s", pkgtype, workdir) From a6cc6efb11cb8db8af151694ddd1e73dd73ff9ce Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 13:26:15 +0100 Subject: [PATCH 043/430] update test_index_functions to only build index for subdirectory with easyconfig files (excl. tempoary log files) --- test/framework/filetools.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 5d127914e7..800ee9ba0b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2421,12 +2421,13 @@ def test_index_functions(self): self.assertTrue(fp.endswith('.eb') or os.path.basename(fp) == 'checksums.json') # set up some files to create actual index file for - ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g')) + ecs_dir = os.path.join(self.test_prefix, 'easyconfigs') + ft.copy_dir(os.path.join(test_ecs, 'g'), ecs_dir) # test dump_index function - index_fp = ft.dump_index(self.test_prefix) + index_fp = ft.dump_index(ecs_dir) self.assertExists(index_fp) - self.assertTrue(os.path.samefile(self.test_prefix, os.path.dirname(index_fp))) + self.assertTrue(os.path.samefile(ecs_dir, os.path.dirname(index_fp))) datestamp_pattern = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+" expected_header = [ @@ -2434,9 +2435,9 @@ def test_index_functions(self): "# valid until: " + datestamp_pattern, ] expected = [ - os.path.join('g', 'gzip', 'gzip-1.4.eb'), - os.path.join('g', 'GCC', 'GCC-7.3.0-2.30.eb'), - os.path.join('g', 'gompic', 'gompic-2018a.eb'), + os.path.join('gzip', 'gzip-1.4.eb'), + os.path.join('GCC', 'GCC-7.3.0-2.30.eb'), + os.path.join('gompic', 'gompic-2018a.eb'), ] index_txt = ft.read_file(index_fp) for fn in expected_header + expected: @@ -2446,28 +2447,28 @@ def test_index_functions(self): # test load_index function self.mock_stderr(True) self.mock_stdout(True) - index = ft.load_index(self.test_prefix) + index = ft.load_index(ecs_dir) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) self.assertFalse(stderr) - regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) - self.assertEqual(len(index), 26) + self.assertEqual(len(index), 25) for fn in expected: self.assertIn(fn, index) # dump_index will not overwrite existing index without force error_pattern = "File exists, not overwriting it without --force" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, self.test_prefix) + self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, ecs_dir) ft.remove_file(index_fp) # test creating index file that's infinitely valid - index_fp = ft.dump_index(self.test_prefix, max_age_sec=0) + index_fp = ft.dump_index(ecs_dir, max_age_sec=0) index_txt = ft.read_file(index_fp) expected_header[1] = r"# valid until: 9999-12-31 23:59:59\.9+" for fn in expected_header + expected: @@ -2476,40 +2477,40 @@ def test_index_functions(self): self.mock_stderr(True) self.mock_stdout(True) - index = ft.load_index(self.test_prefix) + index = ft.load_index(ecs_dir) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) self.assertFalse(stderr) - regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) - self.assertEqual(len(index), 26) + self.assertEqual(len(index), 25) for fn in expected: self.assertIn(fn, index) ft.remove_file(index_fp) # test creating index file that's only valid for a (very) short amount of time - index_fp = ft.dump_index(self.test_prefix, max_age_sec=1) + index_fp = ft.dump_index(ecs_dir, max_age_sec=1) time.sleep(3) self.mock_stderr(True) self.mock_stdout(True) - index = ft.load_index(self.test_prefix) + index = ft.load_index(ecs_dir) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) self.assertIsNone(index) self.assertFalse(stdout) - regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % self.test_prefix) + regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % ecs_dir) self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) # check whether load_index takes into account --ignore-index init_config(build_options={'ignore_index': True}) - self.assertEqual(ft.load_index(self.test_prefix), None) + self.assertEqual(ft.load_index(ecs_dir), None) def test_search_file(self): """Test search_file function.""" From 488d92e571e63256fd1ae94700a28be95d8a5191 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 14:36:02 +0100 Subject: [PATCH 044/430] switch to run_shell_cmd in options test module --- test/framework/options.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 43b5fe66a6..d82b8bf6e9 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -64,7 +64,7 @@ from easybuild.tools.options import EasyBuildOptions, opts_dict_to_eb_opts, parse_external_modules_metadata from easybuild.tools.options import set_up_configuration, set_tmpdir, use_color from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import HAVE_ARCHSPEC from easybuild.tools.version import VERSION from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config @@ -3760,8 +3760,8 @@ def test_include_module_naming_schemes(self): # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): - _, ec = run_cmd("cd %s; python -c 'import easybuild.framework'" % self.test_prefix, log_ok=False) - if ec > 0: + res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) + if res.exit_code != 0: pythonpath = '%s:%s' % (topdir, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -3776,8 +3776,9 @@ def test_include_module_naming_schemes(self): args = ['--avail-module-naming-schemes'] test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertFalse(mns_regex.search(logtxt), "Unexpected pattern '%s' found in: %s" % (mns_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertFalse(mns_regex.search(res.output), + f"Unexpected pattern '{mns_regex.pattern}' found in: {res.output}") # include extra test MNS mns_txt = '\n'.join([ @@ -3793,8 +3794,9 @@ def test_include_module_naming_schemes(self): args.append('--include-module-naming-schemes=%s/*.py' % self.test_prefix) test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertTrue(mns_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (mns_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertTrue(mns_regex.search(res.output), + f"Pattern '{mns_regex.pattern}' *not* found in: {res.output}") def test_use_included_module_naming_scheme(self): """Test using an included module naming scheme.""" @@ -3850,8 +3852,8 @@ def test_include_toolchains(self): # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): - _, ec = run_cmd("cd %s; python -c 'import easybuild.framework'" % self.test_prefix, log_ok=False) - if ec > 0: + res = run_shell_cmd(f"cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) + if res.exit_code != 0: pythonpath = '%s:%s' % (topdir, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -3869,8 +3871,9 @@ def test_include_toolchains(self): args = ['--list-toolchains'] test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertFalse(tc_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (tc_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertFalse(tc_regex.search(res.output), + f"Pattern '{tc_regex.pattern}' *not* found in: {res.output}") # include extra test toolchain comp_txt = '\n'.join([ @@ -3891,8 +3894,9 @@ def test_include_toolchains(self): args.append('--include-toolchains=%s/*.py,%s/*/*.py' % (self.test_prefix, self.test_prefix)) test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertTrue(tc_regex.search(logtxt), "Pattern '%s' found in: %s" % (tc_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertTrue(tc_regex.search(res.output), + f"Pattern '{tc_regex.pattern}' found in: {res.output}") def test_cleanup_tmpdir(self): """Test --cleanup-tmpdir.""" @@ -5164,13 +5168,13 @@ def test_dump_env_script(self): self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) with self.mocked_stdout_stderr(): - out, ec = run_cmd("function module { echo $@; } && source %s && echo FC: $FC" % env_script, simple=False) + res = run_shell_cmd(f"function module {{ echo $@; }} && source {env_script} && echo FC: $FC") expected_out = '\n'.join([ "load GCC/4.6.4", "load hwloc/1.11.8-GCC-4.6.4", "FC: gfortran", ]) - self.assertEqual(out.strip(), expected_out) + self.assertEqual(res.output.strip(), expected_out) def test_stop(self): """Test use of --stop.""" From 060b9c3d0e3d00658ee66f432d5616701cd15e65 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 14:41:39 +0100 Subject: [PATCH 045/430] switch to run_shell_cmd in remaining test modules --- test/framework/package.py | 2 +- test/framework/repository.py | 6 +++--- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 4 ++-- test/framework/sandbox/easybuild/easyblocks/l/libtoy.py | 8 ++++---- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 8 ++++---- test/framework/toy_build.py | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/framework/package.py b/test/framework/package.py index 38f24242a7..7298d6ecc9 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -44,7 +44,7 @@ FPM_OUTPUT_FILE = 'fpm_mocked.out' -# purposely using non-bash script, to detect issues with shebang line being ignored (run_cmd with shell=False) +# purposely using non-bash script, to detect issues with shebang line being ignored (run_shell_cmd with use_bash=False) MOCKED_FPM = """#!/usr/bin/env python import os, sys diff --git a/test/framework/repository.py b/test/framework/repository.py index a10cfeebf8..acc2749c55 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -43,7 +43,7 @@ from easybuild.tools.repository.hgrepo import HgRepository from easybuild.tools.repository.svnrepo import SvnRepository from easybuild.tools.repository.repository import init_repository -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.version import VERSION @@ -95,10 +95,10 @@ def test_gitrepo(self): tmpdir = tempfile.mkdtemp() cmd = "cd %s && git clone --bare %s" % (tmpdir, test_repo_url) with self.mocked_stdout_stderr(): - _, ec = run_cmd(cmd, simple=False, log_all=False, log_ok=False) + res = run_shell_cmd(cmd, fail_on_error=False) # skip remainder of test if creating bare git repo didn't work - if ec == 0: + if res.exit_code == 0: repo = GitRepository(os.path.join(tmpdir, 'testrepository.git')) repo.init() toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 9c700cf779..de8c4c89c1 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -32,7 +32,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class Toy_Extension(ExtensionEasyBlock): @@ -67,7 +67,7 @@ def run(self, *args, **kwargs): EB_toy.build_step(self.master, name=self.name, cfg=self.cfg) if self.cfg['toy_ext_param']: - run_cmd(self.cfg['toy_ext_param']) + run_shell_cmd(self.cfg['toy_ext_param']) return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name) diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py index 50b573649a..2c0ba5a9e5 100644 --- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -30,7 +30,7 @@ import os from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext SHLIB_EXT = get_shared_lib_ext() @@ -40,7 +40,7 @@ class EB_libtoy(EasyBlock): """Support for building/installing libtoy.""" def banned_linked_shared_libs(self): - default = '/thiswillnotbethere,libtoytoytoy.%s,toytoytoy' % SHLIB_EXT + default = f'/thiswillnotbethere,libtoytoytoy.{SHLIB_EXT},toytoytoy' return os.getenv('EB_LIBTOY_BANNED_SHARED_LIBS', default).split(',') def required_linked_shared_libs(self): @@ -53,8 +53,8 @@ def configure_step(self, name=None): def build_step(self, name=None, buildopts=None): """Build libtoy.""" - run_cmd('make') + run_shell_cmd('make') def install_step(self, name=None): """Install libtoy.""" - run_cmd('make install PREFIX="%s"' % self.installdir) + run_shell_cmd(f'make install PREFIX="{self.installdir}"') diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index c4614e3333..a1454435e3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -38,7 +38,7 @@ from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts): @@ -108,7 +108,7 @@ def configure_step(self, name=None, cfg=None): 'echo "Configured"', cfg['configopts'] ]) - run_cmd(cmd) + run_shell_cmd(cmd) if os.path.exists("%s.source" % name): os.rename('%s.source' % name, '%s.c' % name) @@ -124,8 +124,8 @@ def build_step(self, name=None, cfg=None): cmd = compose_toy_build_cmd(self.cfg, name, cfg['prebuildopts'], cfg['buildopts']) # purposely run build command without checking exit code; # we rely on this in test_toy_build_hooks - (out, ec) = run_cmd(cmd, log_ok=False, log_all=False) - if ec: + res = run_shell_cmd(cmd, fail_on_error=False) + if res.exit_code: print_warning("Command '%s' failed, but we'll ignore it..." % cmd) def install_step(self, name=None): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d4c7988534..ca53efc610 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2963,7 +2963,7 @@ def test_toy_build_trace(self): r"^ >> running command:", r"\t\[started at: .*\]", r"\t\[working dir: .*\]", - r"\t\[output logged in .*\]", + r"\t\[output saved to .*\]", r"\tgcc toy.c -o toy\n" r'', ]), From 59ad5ed408ba134b9140abec4e743aeef53d5d85 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 15:23:06 +0100 Subject: [PATCH 046/430] deprcate run_cmd and run_cmd_qa & co, move them to easybuild._deprecated module --- easybuild/_deprecated.py | 811 ++++++++++++++++++++++++++++++++++ easybuild/base/fancylogger.py | 2 +- easybuild/tools/build_log.py | 2 +- easybuild/tools/run.py | 750 +------------------------------ 4 files changed, 828 insertions(+), 737 deletions(-) create mode 100644 easybuild/_deprecated.py diff --git a/easybuild/_deprecated.py b/easybuild/_deprecated.py new file mode 100644 index 0000000000..f22ad0ba53 --- /dev/null +++ b/easybuild/_deprecated.py @@ -0,0 +1,811 @@ +# # +# Copyright 2023-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Deprecated functionality, which will be removed with next major EasyBuild version + +Authors: + +* Kenneth Hoste (Ghent University) +""" +import contextlib +import functools +import os +import re +import signal +import subprocess +import sys +import tempfile +import time +from datetime import datetime + +import easybuild.tools.asyncprocess as asyncprocess +from easybuild.base import fancylogger +from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since +from easybuild.tools.config import ERROR, IGNORE, WARN, build_option +from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook +from easybuild.tools.utilities import nub, trace_msg + + +_log = fancylogger.getLogger('_deprecated', fname=False) + + +errors_found_in_log = 0 + +# default strictness level +strictness = WARN + + +CACHED_COMMANDS = [ + "sysctl -n hw.cpufrequency_max", # used in get_cpu_speed (OS X) + "sysctl -n hw.memsize", # used in get_total_memory (OS X) + "sysctl -n hw.ncpu", # used in get_avail_core_count (OS X) + "sysctl -n machdep.cpu.brand_string", # used in get_cpu_model (OS X) + "sysctl -n machdep.cpu.vendor", # used in get_cpu_vendor (OS X) + "type module", # used in ModulesTool.check_module_function + "type _module_raw", # used in EnvironmentModules.check_module_function + "ulimit -u", # used in det_parallelism +] + + +def run_cmd_cache(func): + """Function decorator to cache (and retrieve cached) results of running commands.""" + cache = {} + + @functools.wraps(func) + def cache_aware_func(cmd, *args, **kwargs): + """Retrieve cached result of selected commands, or run specified and collect & cache result.""" + + # cache key is combination of command and input provided via stdin ('inp' named option) + key = (cmd, kwargs.get('inp', None)) + # fetch from cache if available, cache it if it's not, but only on cmd strings + if isinstance(cmd, str) and key in cache: + _log.debug("Using cached value for command '%s': %s", cmd, cache[key]) + return cache[key] + else: + res = func(cmd, *args, **kwargs) + if cmd in CACHED_COMMANDS: + cache[key] = res + return res + + # expose clear/update methods of cache to wrapped function + cache_aware_func.clear_cache = cache.clear + cache_aware_func.update_cache = cache.update + + return cache_aware_func + + +def get_output_from_process(proc, read_size=None, asynchronous=False): + """ + Get output from running process (that was opened with subprocess.Popen). + + :param proc: process to get output from + :param read_size: number of bytes of output to read (if None: read all output) + :param asynchronous: get output asynchronously + """ + + if asynchronous: + # e=False is set to avoid raising an exception when command has completed; + # that's needed to ensure we get all output, + # see https://github.com/easybuilders/easybuild-framework/issues/3593 + output = asyncprocess.recv_some(proc, e=False) + elif read_size: + output = proc.stdout.read(read_size) + else: + output = proc.stdout.read() + + # need to be careful w.r.t. encoding since we want to obtain a string value, + # and the output may include non UTF-8 characters + # * in Python 2, .decode() returns a value of type 'unicode', + # but we really want a regular 'str' value (which is also why we use 'ignore' for encoding errors) + # * in Python 3, .decode() returns a 'str' value when called on the 'bytes' value obtained from .read() + output = str(output.decode('ascii', 'ignore')) + + return output + + +@run_cmd_cache +def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, + force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False, + with_hooks=True): + """ + Run specified command (in a subshell) + :param cmd: command to run + :param log_ok: only run output/exit code for failing commands (exit code non-zero) + :param log_all: always log command output and exit code + :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) + :param inp: the input given to the command via stdin + :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) + :param log_output: indicate whether all output of command should be logged to a separate temporary logfile + :param path: path to execute the command in; current working directory is used if unspecified + :param force_in_dry_run: force running the command during dry run + :param verbose: include message on running the command in dry run output + :param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True + :param trace: print command being executed as part of trace output + :param stream_output: enable streaming command output to stdout + :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) + :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) + """ + + _log.deprecated("run_cmd is deprecated, use run_shell_cmd from easybuild.tools.run instead", '6.0') + + cwd = os.getcwd() + + if isinstance(cmd, str): + cmd_msg = cmd.strip() + elif isinstance(cmd, list): + cmd_msg = ' '.join(cmd) + else: + raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd) + + if shell is None: + shell = True + if isinstance(cmd, list): + raise EasyBuildError("When passing cmd as a list then `shell` must be set explictely! " + "Note that all elements of the list but the first are treated as arguments " + "to the shell and NOT to the command to be executed!") + + if log_output or (trace and build_option('trace')): + # collect output of running command in temporary log file, if desired + fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-') + os.close(fd) + try: + cmd_log = open(cmd_log_fn, 'w') + except IOError as err: + raise EasyBuildError("Failed to open temporary log file for output of command: %s", err) + _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) + else: + cmd_log_fn, cmd_log = None, None + + # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely + if stream_output is None and build_option('logtostdout'): + _log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg) + stream_output = True + + if stream_output: + print_msg("(streaming) output for command '%s':" % cmd_msg) + + start_time = datetime.now() + if trace: + trace_txt = "running command:\n" + trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') + trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd()) + if inp: + trace_txt += "\t[input: %s]\n" % inp + trace_txt += "\t[output logged in %s]\n" % cmd_log_fn + trace_msg(trace_txt + '\t' + cmd_msg) + + # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced) + if not force_in_dry_run and build_option('extended_dry_run'): + if path is None: + path = cwd + if verbose: + dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent')) + dry_run_msg(" (in %s)" % path, silent=build_option('silent')) + + # make sure we get the type of the return value right + if simple: + return True + else: + # output, exit code + return ('', 0) + + try: + if path: + os.chdir(path) + + _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd())) + except OSError as err: + _log.warning("Failed to change to %s: %s" % (path, err)) + _log.info("running cmd %s in non-existing directory, might fail!", cmd) + + if cmd_log: + cmd_log.write("# output for command: %s\n\n" % cmd_msg) + + exec_cmd = "/bin/bash" + + if not shell: + if isinstance(cmd, list): + exec_cmd = None + cmd.insert(0, '/usr/bin/env') + elif isinstance(cmd, str): + cmd = '/usr/bin/env %s' % cmd + else: + raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) + + if with_hooks: + hooks = load_hooks(build_option('hooks')) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()}) + if isinstance(hook_res, str): + cmd, old_cmd = hook_res, cmd + _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) + + _log.info('running cmd: %s ' % cmd) + try: + proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd) + except OSError as err: + raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) + + if inp: + proc.stdin.write(inp.encode()) + proc.stdin.close() + + if asynchronous: + return (proc, cmd, cwd, start_time, cmd_log) + else: + return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple, + regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks) + + +def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''): + """ + Check status of command that was started asynchronously. + + :param proc: subprocess.Popen instance representing asynchronous command + :param cmd: command being run + :param owd: original working directory + :param start_time: start time of command (datetime instance) + :param cmd_log: log file to print command output to + :param fail_on_error: raise EasyBuildError when command exited with an error + :param output_read_size: number of bytes to read from output + :param output: already collected output for this command + + :result: dict value with result of the check (boolean 'done', 'exit_code', 'output') + """ + + # use small read size, to avoid waiting for a long time until sufficient output is produced + if output_read_size: + if not isinstance(output_read_size, int) or output_read_size < 0: + raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)") + add_out = get_output_from_process(proc, read_size=output_read_size) + _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out)) + output += add_out + + exit_code = proc.poll() + if exit_code is None: + _log.debug("Asynchronous command '%s' still running..." % cmd) + done = False + else: + _log.debug("Asynchronous command '%s' completed!", cmd) + output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, + simple=False, trace=False, log_ok=fail_on_error) + done = True + + res = { + 'done': done, + 'exit_code': exit_code, + 'output': output, + } + return res + + +def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, + regexp=True, stream_output=None, trace=True, output='', with_hook=True): + """ + Complete running of command represented by passed subprocess.Popen instance. + + :param proc: subprocess.Popen instance representing running command + :param cmd: command being run + :param owd: original working directory + :param start_time: start time of command (datetime instance) + :param cmd_log: log file to print command output to + :param log_ok: only run output/exit code for failing commands (exit code non-zero) + :param log_all: always log command output and exit code + :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) + :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) + :param stream_output: enable streaming command output to stdout + :param trace: print command being executed as part of trace output + :param with_hook: trigger post run_shell_cmd hooks (if defined) + """ + # use small read size when streaming output, to make it stream more fluently + # read size should not be too small though, to avoid too much overhead + if stream_output: + read_size = 128 + else: + read_size = 1024 * 8 + + stdouterr = output + + try: + ec = proc.poll() + while ec is None: + # need to read from time to time. + # - otherwise the stdout/stderr buffer gets filled and it all stops working + output = get_output_from_process(proc, read_size=read_size) + if cmd_log: + cmd_log.write(output) + if stream_output: + sys.stdout.write(output) + stdouterr += output + ec = proc.poll() + + # read remaining data (all of it) + output = get_output_from_process(proc) + finally: + proc.stdout.close() + + if cmd_log: + cmd_log.write(output) + cmd_log.close() + if stream_output: + sys.stdout.write(output) + stdouterr += output + + if with_hook: + hooks = load_hooks(build_option('hooks')) + run_hook_kwargs = { + 'exit_code': ec, + 'output': stdouterr, + 'work_dir': os.getcwd(), + } + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) + + if trace: + trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) + + try: + os.chdir(owd) + except OSError as err: + raise EasyBuildError("Failed to return to %s after executing command: %s", owd, err) + + return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp) + + +def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, + maxhits=50, trace=True): + """ + Run specified interactive command (in a subshell) + :param cmd: command to run + :param qa: dictionary which maps question to answers + :param no_qa: list of patters that are not questions + :param log_ok: only run output/exit code for failing commands (exit code non-zero) + :param log_all: always log command output and exit code + :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) + :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) + :param std_qa: dictionary which maps question regex patterns to answers + :param path: path to execute the command is; current working directory is used if unspecified + :param maxhits: maximum number of cycles (seconds) without being able to find a known question + :param trace: print command being executed as part of trace output + """ + cwd = os.getcwd() + + if not isinstance(cmd, str) and len(cmd) > 1: + # We use shell=True and hence we should really pass the command as a string + # When using a list then every element past the first is passed to the shell itself, not the command! + raise EasyBuildError("The command passed must be a string!") + + if log_all or (trace and build_option('trace')): + # collect output of running command in temporary log file, if desired + fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-') + os.close(fd) + try: + cmd_log = open(cmd_log_fn, 'w') + except IOError as err: + raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err) + _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) + else: + cmd_log_fn, cmd_log = None, None + + start_time = datetime.now() + if trace: + trace_txt = "running interactive command:\n" + trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') + trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd()) + trace_txt += "\t[output logged in %s]\n" % cmd_log_fn + trace_msg(trace_txt + '\t' + cmd.strip()) + + # early exit in 'dry run' mode, after printing the command that would be run + if build_option('extended_dry_run'): + if path is None: + path = cwd + dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent')) + dry_run_msg(" (in %s)" % path, silent=build_option('silent')) + if cmd_log: + cmd_log.close() + if simple: + return True + else: + # output, exit code + return ('', 0) + + try: + if path: + os.chdir(path) + + _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd())) + except OSError as err: + _log.warning("Failed to change to %s: %s" % (path, err)) + _log.info("running cmd %s in non-existing directory, might fail!" % cmd) + + # Part 1: process the QandA dictionary + # given initial set of Q and A (in dict), return dict of reg. exp. and A + # + # make regular expression that matches the string with + # - replace whitespace + # - replace newline + + def escape_special(string): + return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string) + + split = r'[\s\n]+' + regSplit = re.compile(r"" + split) + + def process_QA(q, a_s): + splitq = [escape_special(x) for x in regSplit.split(q)] + regQtxt = split.join(splitq) + split.rstrip('+') + "*$" + # add optional split at the end + for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]: + a_s[i] += '\n' + regQ = re.compile(r"" + regQtxt) + if regQ.search(q): + return (a_s, regQ) + else: + raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt) + + def check_answers_list(answers): + """Make sure we have a list of answers (as strings).""" + if isinstance(answers, str): + answers = [answers] + elif not isinstance(answers, list): + if cmd_log: + cmd_log.close() + raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)", + question, type(answers), answers) + # list is manipulated when answering matching question, so return a copy + return answers[:] + + new_qa = {} + _log.debug("new_qa: ") + for question, answers in qa.items(): + answers = check_answers_list(answers) + (answers, regQ) = process_QA(question, answers) + new_qa[regQ] = answers + _log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ])) + + new_std_qa = {} + if std_qa: + for question, answers in std_qa.items(): + regQ = re.compile(r"" + question + r"[\s\n]*$") + answers = check_answers_list(answers) + for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]: + answers[i] += '\n' + new_std_qa[regQ] = answers + _log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ])) + + new_no_qa = [] + if no_qa: + # simple statements, can contain wildcards + new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa] + + _log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa]) + + # Part 2: Run the command and answer questions + # - this needs asynchronous stdout + + hooks = load_hooks(build_option('hooks')) + run_hook_kwargs = { + 'interactive': True, + 'work_dir': os.getcwd(), + } + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) + if isinstance(hook_res, str): + cmd, old_cmd = hook_res, cmd + _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", + RUN_SHELL_CMD, cmd, old_cmd) + + # # Log command output + if cmd_log: + cmd_log.write("# output for interactive command: %s\n\n" % cmd) + + # Make sure we close the proc handles and the cmd_log file + @contextlib.contextmanager + def get_proc(): + try: + proc = asyncprocess.Popen(cmd, shell=True, stdout=asyncprocess.PIPE, stderr=asyncprocess.STDOUT, + stdin=asyncprocess.PIPE, close_fds=True, executable='/bin/bash') + except OSError as err: + if cmd_log: + cmd_log.close() + raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err) + try: + yield proc + finally: + if proc.stdout: + proc.stdout.close() + if proc.stdin: + proc.stdin.close() + if cmd_log: + cmd_log.close() + + with get_proc() as proc: + ec = proc.poll() + stdout_err = '' + old_len_out = -1 + hit_count = 0 + + while ec is None: + # need to read from time to time. + # - otherwise the stdout/stderr buffer gets filled and it all stops working + try: + out = get_output_from_process(proc, asynchronous=True) + + if cmd_log: + cmd_log.write(out) + stdout_err += out + # recv_some used by get_output_from_process for getting asynchronous output may throw exception + except (IOError, Exception) as err: + _log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err) + out = None + + hit = False + for question, answers in new_qa.items(): + res = question.search(stdout_err) + if out and res: + fa = answers[0] % res.groupdict() + # cycle through list of answers + last_answer = answers.pop(0) + answers.append(last_answer) + _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers) + + _log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:]) + asyncprocess.send_all(proc, fa) + hit = True + break + if not hit: + for question, answers in new_std_qa.items(): + res = question.search(stdout_err) + if out and res: + fa = answers[0] % res.groupdict() + # cycle through list of answers + last_answer = answers.pop(0) + answers.append(last_answer) + _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers) + + _log.debug("run_cmd_qa answer %s std question %s out %s", + fa, question.pattern, stdout_err[-50:]) + asyncprocess.send_all(proc, fa) + hit = True + break + if not hit: + if len(stdout_err) > old_len_out: + old_len_out = len(stdout_err) + else: + noqa = False + for r in new_no_qa: + if r.search(stdout_err): + _log.debug("runqanda: noQandA found for out %s", stdout_err[-50:]) + noqa = True + if not noqa: + hit_count += 1 + else: + hit_count = 0 + else: + hit_count = 0 + + if hit_count > maxhits: + # explicitly kill the child process before exiting + try: + os.killpg(proc.pid, signal.SIGKILL) + os.kill(proc.pid, signal.SIGKILL) + except OSError as err: + _log.debug("run_cmd_qa exception caught when killing child process: %s", err) + _log.debug("run_cmd_qa: full stdouterr: %s", stdout_err) + raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s", + cmd, maxhits, stdout_err[-500:]) + + # the sleep below is required to avoid exiting on unknown 'questions' too early (see above) + time.sleep(1) + ec = proc.poll() + + # Process stopped. Read all remaining data + try: + if proc.stdout: + out = get_output_from_process(proc) + stdout_err += out + if cmd_log: + cmd_log.write(out) + except IOError as err: + _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err) + + run_hook_kwargs.update({ + 'interactive': True, + 'exit_code': ec, + 'output': stdout_err, + }) + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) + + if trace: + trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) + + try: + os.chdir(cwd) + except OSError as err: + raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) + + return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp) + + +def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): + """ + Parse command output and construct return value. + :param cmd: executed command + :param stdouterr: combined stdout/stderr of executed command + :param ec: exit code of executed command + :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) + :param log_all: always log command output and exit code + :param log_ok: only run output/exit code for failing commands (exit code non-zero) + :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) + """ + if strictness == IGNORE: + check_ec = False + fail_on_error_match = False + elif strictness == WARN: + check_ec = True + fail_on_error_match = False + elif strictness == ERROR: + check_ec = True + fail_on_error_match = True + else: + raise EasyBuildError("invalid strictness setting: %s", strictness) + + # allow for overriding the regexp setting + if not regexp: + fail_on_error_match = False + + if ec and (log_all or log_ok): + # We don't want to error if the user doesn't care + if check_ec: + raise EasyBuildError('cmd "%s" exited with exit code %s and output:\n%s', cmd, ec, stdouterr) + else: + _log.warning('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr)) + elif not ec: + if log_all: + _log.info('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr)) + else: + _log.debug('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr)) + + # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in + if fail_on_error_match or regexp: + res = parse_log_for_error(stdouterr, regexp, stdout=False) + if res: + errors = "\n\t" + "\n\t".join([r[0] for r in res]) + error_str = "error" if len(res) == 1 else "errors" + if fail_on_error_match: + raise EasyBuildError("Found %s %s in output of %s:%s", len(res), error_str, cmd, errors) + else: + _log.warning("Found %s potential %s (some may be harmless) in output of %s:%s", + len(res), error_str, cmd, errors) + + if simple: + if ec: + # If the user does not care -> will return true + return not check_ec + else: + return True + else: + # Because we are not running in simple mode, we return the output and ec to the user + return (stdouterr, ec) + + +def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): + """ + txt is multiline string. + - in memory + regExp is a one-line regular expression + - default + """ + global errors_found_in_log + + if regExp and isinstance(regExp, bool): + regExp = r"(?= loose_mv.version[:depth]: self.raiseException("DEPRECATED (since v%s) functionality used: %s" % (max_ver, msg), exception=exception) else: - deprecation_msg = "Deprecated functionality, will no longer work in v%s: %s" % (max_ver, msg) + deprecation_msg = "Deprecated functionality, will no longer work in EasyBuild v%s: %s" % (max_ver, msg) log_callback(deprecation_msg) def _handleFunction(self, function, levelno, **kwargs): diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index f8529fff57..df2bf829aa 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -55,7 +55,7 @@ # allow some experimental experimental code EXPERIMENTAL = False -DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html' +DEPRECATED_DOC_URL = 'https://docs.easybuild.io/deprecated-functionality' DRY_RUN_BUILD_DIR = None DRY_RUN_SOFTWARE_INSTALL_DIR = None diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 0bb0d3c9f4..1edca73364 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -35,37 +35,31 @@ * Toon Willems (Ghent University) * Ward Poelmans (Ghent University) """ -import contextlib import functools import inspect import os -import re -import signal import subprocess import sys import tempfile -import time from collections import namedtuple from datetime import datetime -import easybuild.tools.asyncprocess as asyncprocess +# import deprecated functions so they can still be imported from easybuild.tools.run for now +from easybuild._deprecated import check_async_cmd, check_log_for_errors, complete_cmd, extract_errors_from_log # noqa +from easybuild._deprecated import get_output_from_process, parse_cmd_output, parse_log_for_error, run_cmd # noqa +from easybuild._deprecated import run_cmd_cache, run_cmd_qa # noqa + from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since -from easybuild.tools.config import ERROR, IGNORE, WARN, build_option +from easybuild.tools.config import build_option from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook -from easybuild.tools.utilities import nub, trace_msg +from easybuild.tools.utilities import trace_msg _log = fancylogger.getLogger('run', fname=False) -errors_found_in_log = 0 - -# default strictness level -strictness = WARN - - -CACHED_COMMANDS = [ +CACHED_COMMANDS = ( "sysctl -n hw.cpufrequency_max", # used in get_cpu_speed (OS X) "sysctl -n hw.memsize", # used in get_total_memory (OS X) "sysctl -n hw.ncpu", # used in get_avail_core_count (OS X) @@ -74,7 +68,7 @@ "type module", # used in ModulesTool.check_module_function "type _module_raw", # used in EnvironmentModules.check_module_function "ulimit -u", # used in det_parallelism -] +) RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', @@ -142,7 +136,7 @@ def raise_run_shell_cmd_error(cmd_res): # need to go 3 levels down: # 1) this function # 2) run_shell_cmd function - # 3) run_cmd_cache decorator + # 3) run_shell_cmd_cache decorator # 4) actual caller site frameinfo = inspect.getouterframes(inspect.currentframe())[3] caller_info = (frameinfo.filename, frameinfo.lineno, frameinfo.function) @@ -150,15 +144,15 @@ def raise_run_shell_cmd_error(cmd_res): raise RunShellCmdError(cmd_res, caller_info) -def run_cmd_cache(func): +def run_shell_cmd_cache(func): """Function decorator to cache (and retrieve cached) results of running commands.""" cache = {} @functools.wraps(func) def cache_aware_func(cmd, *args, **kwargs): """Retrieve cached result of selected commands, or run specified and collect & cache result.""" - # cache key is combination of command and input provided via stdin ('stdin' for run, 'inp' for run_cmd) - key = (cmd, kwargs.get('stdin', None) or kwargs.get('inp', None)) + # cache key is combination of command and input provided via stdin + key = (cmd, kwargs.get('stdin', None)) # fetch from cache if available, cache it if it's not, but only on cmd strings if isinstance(cmd, str) and key in cache: _log.debug("Using cached value for command '%s': %s", cmd, cache[key]) @@ -176,9 +170,6 @@ def cache_aware_func(cmd, *args, **kwargs): return cache_aware_func -run_shell_cmd_cache = run_cmd_cache - - @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, @@ -245,10 +236,10 @@ def to_cmd_str(cmd): os.makedirs(toptmpdir, exist_ok=True) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') cmd_out_fp = os.path.join(tmpdir, 'out.txt') - _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') + _log.info(f'run_shell_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') if split_stderr: cmd_err_fp = os.path.join(tmpdir, 'err.txt') - _log.info(f'run_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') + _log.info(f'run_shell_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') else: cmd_err_fp = None else: @@ -397,717 +388,6 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): trace_msg('\n'.join(lines)) -def get_output_from_process(proc, read_size=None, asynchronous=False): - """ - Get output from running process (that was opened with subprocess.Popen). - - :param proc: process to get output from - :param read_size: number of bytes of output to read (if None: read all output) - :param asynchronous: get output asynchronously - """ - - if asynchronous: - # e=False is set to avoid raising an exception when command has completed; - # that's needed to ensure we get all output, - # see https://github.com/easybuilders/easybuild-framework/issues/3593 - output = asyncprocess.recv_some(proc, e=False) - elif read_size: - output = proc.stdout.read(read_size) - else: - output = proc.stdout.read() - - # need to be careful w.r.t. encoding since we want to obtain a string value, - # and the output may include non UTF-8 characters - # * in Python 2, .decode() returns a value of type 'unicode', - # but we really want a regular 'str' value (which is also why we use 'ignore' for encoding errors) - # * in Python 3, .decode() returns a 'str' value when called on the 'bytes' value obtained from .read() - output = str(output.decode('ascii', 'ignore')) - - return output - - -@run_cmd_cache -def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, - force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False, - with_hooks=True): - """ - Run specified command (in a subshell) - :param cmd: command to run - :param log_ok: only run output/exit code for failing commands (exit code non-zero) - :param log_all: always log command output and exit code - :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) - :param inp: the input given to the command via stdin - :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) - :param log_output: indicate whether all output of command should be logged to a separate temporary logfile - :param path: path to execute the command in; current working directory is used if unspecified - :param force_in_dry_run: force running the command during dry run - :param verbose: include message on running the command in dry run output - :param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True - :param trace: print command being executed as part of trace output - :param stream_output: enable streaming command output to stdout - :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) - :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) - """ - cwd = os.getcwd() - - if isinstance(cmd, str): - cmd_msg = cmd.strip() - elif isinstance(cmd, list): - cmd_msg = ' '.join(cmd) - else: - raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd) - - if shell is None: - shell = True - if isinstance(cmd, list): - raise EasyBuildError("When passing cmd as a list then `shell` must be set explictely! " - "Note that all elements of the list but the first are treated as arguments " - "to the shell and NOT to the command to be executed!") - - if log_output or (trace and build_option('trace')): - # collect output of running command in temporary log file, if desired - fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-') - os.close(fd) - try: - cmd_log = open(cmd_log_fn, 'w') - except IOError as err: - raise EasyBuildError("Failed to open temporary log file for output of command: %s", err) - _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) - else: - cmd_log_fn, cmd_log = None, None - - # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely - if stream_output is None and build_option('logtostdout'): - _log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg) - stream_output = True - - if stream_output: - print_msg("(streaming) output for command '%s':" % cmd_msg) - - start_time = datetime.now() - if trace: - trace_txt = "running command:\n" - trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') - trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd()) - if inp: - trace_txt += "\t[input: %s]\n" % inp - trace_txt += "\t[output logged in %s]\n" % cmd_log_fn - trace_msg(trace_txt + '\t' + cmd_msg) - - # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced) - if not force_in_dry_run and build_option('extended_dry_run'): - if path is None: - path = cwd - if verbose: - dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent')) - dry_run_msg(" (in %s)" % path, silent=build_option('silent')) - - # make sure we get the type of the return value right - if simple: - return True - else: - # output, exit code - return ('', 0) - - try: - if path: - os.chdir(path) - - _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd())) - except OSError as err: - _log.warning("Failed to change to %s: %s" % (path, err)) - _log.info("running cmd %s in non-existing directory, might fail!", cmd) - - if cmd_log: - cmd_log.write("# output for command: %s\n\n" % cmd_msg) - - exec_cmd = "/bin/bash" - - if not shell: - if isinstance(cmd, list): - exec_cmd = None - cmd.insert(0, '/usr/bin/env') - elif isinstance(cmd, str): - cmd = '/usr/bin/env %s' % cmd - else: - raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) - - if with_hooks: - hooks = load_hooks(build_option('hooks')) - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()}) - if isinstance(hook_res, str): - cmd, old_cmd = hook_res, cmd - _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) - - _log.info('running cmd: %s ' % cmd) - try: - proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd) - except OSError as err: - raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) - - if inp: - proc.stdin.write(inp.encode()) - proc.stdin.close() - - if asynchronous: - return (proc, cmd, cwd, start_time, cmd_log) - else: - return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple, - regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks) - - -def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''): - """ - Check status of command that was started asynchronously. - - :param proc: subprocess.Popen instance representing asynchronous command - :param cmd: command being run - :param owd: original working directory - :param start_time: start time of command (datetime instance) - :param cmd_log: log file to print command output to - :param fail_on_error: raise EasyBuildError when command exited with an error - :param output_read_size: number of bytes to read from output - :param output: already collected output for this command - - :result: dict value with result of the check (boolean 'done', 'exit_code', 'output') - """ - # use small read size, to avoid waiting for a long time until sufficient output is produced - if output_read_size: - if not isinstance(output_read_size, int) or output_read_size < 0: - raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)") - add_out = get_output_from_process(proc, read_size=output_read_size) - _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out)) - output += add_out - - exit_code = proc.poll() - if exit_code is None: - _log.debug("Asynchronous command '%s' still running..." % cmd) - done = False - else: - _log.debug("Asynchronous command '%s' completed!", cmd) - output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, - simple=False, trace=False, log_ok=fail_on_error) - done = True - - res = { - 'done': done, - 'exit_code': exit_code, - 'output': output, - } - return res - - -def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, - regexp=True, stream_output=None, trace=True, output='', with_hook=True): - """ - Complete running of command represented by passed subprocess.Popen instance. - - :param proc: subprocess.Popen instance representing running command - :param cmd: command being run - :param owd: original working directory - :param start_time: start time of command (datetime instance) - :param cmd_log: log file to print command output to - :param log_ok: only run output/exit code for failing commands (exit code non-zero) - :param log_all: always log command output and exit code - :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) - :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) - :param stream_output: enable streaming command output to stdout - :param trace: print command being executed as part of trace output - :param with_hook: trigger post run_shell_cmd hooks (if defined) - """ - # use small read size when streaming output, to make it stream more fluently - # read size should not be too small though, to avoid too much overhead - if stream_output: - read_size = 128 - else: - read_size = 1024 * 8 - - stdouterr = output - - try: - ec = proc.poll() - while ec is None: - # need to read from time to time. - # - otherwise the stdout/stderr buffer gets filled and it all stops working - output = get_output_from_process(proc, read_size=read_size) - if cmd_log: - cmd_log.write(output) - if stream_output: - sys.stdout.write(output) - stdouterr += output - ec = proc.poll() - - # read remaining data (all of it) - output = get_output_from_process(proc) - finally: - proc.stdout.close() - - if cmd_log: - cmd_log.write(output) - cmd_log.close() - if stream_output: - sys.stdout.write(output) - stdouterr += output - - if with_hook: - hooks = load_hooks(build_option('hooks')) - run_hook_kwargs = { - 'exit_code': ec, - 'output': stdouterr, - 'work_dir': os.getcwd(), - } - run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) - - if trace: - trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) - - try: - os.chdir(owd) - except OSError as err: - raise EasyBuildError("Failed to return to %s after executing command: %s", owd, err) - - return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp) - - -def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, - maxhits=50, trace=True): - """ - Run specified interactive command (in a subshell) - :param cmd: command to run - :param qa: dictionary which maps question to answers - :param no_qa: list of patters that are not questions - :param log_ok: only run output/exit code for failing commands (exit code non-zero) - :param log_all: always log command output and exit code - :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) - :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) - :param std_qa: dictionary which maps question regex patterns to answers - :param path: path to execute the command is; current working directory is used if unspecified - :param maxhits: maximum number of cycles (seconds) without being able to find a known question - :param trace: print command being executed as part of trace output - """ - cwd = os.getcwd() - - if not isinstance(cmd, str) and len(cmd) > 1: - # We use shell=True and hence we should really pass the command as a string - # When using a list then every element past the first is passed to the shell itself, not the command! - raise EasyBuildError("The command passed must be a string!") - - if log_all or (trace and build_option('trace')): - # collect output of running command in temporary log file, if desired - fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-') - os.close(fd) - try: - cmd_log = open(cmd_log_fn, 'w') - except IOError as err: - raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err) - _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) - else: - cmd_log_fn, cmd_log = None, None - - start_time = datetime.now() - if trace: - trace_txt = "running interactive command:\n" - trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') - trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd()) - trace_txt += "\t[output logged in %s]\n" % cmd_log_fn - trace_msg(trace_txt + '\t' + cmd.strip()) - - # early exit in 'dry run' mode, after printing the command that would be run - if build_option('extended_dry_run'): - if path is None: - path = cwd - dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent')) - dry_run_msg(" (in %s)" % path, silent=build_option('silent')) - if cmd_log: - cmd_log.close() - if simple: - return True - else: - # output, exit code - return ('', 0) - - try: - if path: - os.chdir(path) - - _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd())) - except OSError as err: - _log.warning("Failed to change to %s: %s" % (path, err)) - _log.info("running cmd %s in non-existing directory, might fail!" % cmd) - - # Part 1: process the QandA dictionary - # given initial set of Q and A (in dict), return dict of reg. exp. and A - # - # make regular expression that matches the string with - # - replace whitespace - # - replace newline - - def escape_special(string): - return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string) - - split = r'[\s\n]+' - regSplit = re.compile(r"" + split) - - def process_QA(q, a_s): - splitq = [escape_special(x) for x in regSplit.split(q)] - regQtxt = split.join(splitq) + split.rstrip('+') + "*$" - # add optional split at the end - for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]: - a_s[i] += '\n' - regQ = re.compile(r"" + regQtxt) - if regQ.search(q): - return (a_s, regQ) - else: - raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt) - - def check_answers_list(answers): - """Make sure we have a list of answers (as strings).""" - if isinstance(answers, str): - answers = [answers] - elif not isinstance(answers, list): - if cmd_log: - cmd_log.close() - raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)", - question, type(answers), answers) - # list is manipulated when answering matching question, so return a copy - return answers[:] - - new_qa = {} - _log.debug("new_qa: ") - for question, answers in qa.items(): - answers = check_answers_list(answers) - (answers, regQ) = process_QA(question, answers) - new_qa[regQ] = answers - _log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ])) - - new_std_qa = {} - if std_qa: - for question, answers in std_qa.items(): - regQ = re.compile(r"" + question + r"[\s\n]*$") - answers = check_answers_list(answers) - for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]: - answers[i] += '\n' - new_std_qa[regQ] = answers - _log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ])) - - new_no_qa = [] - if no_qa: - # simple statements, can contain wildcards - new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa] - - _log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa]) - - # Part 2: Run the command and answer questions - # - this needs asynchronous stdout - - hooks = load_hooks(build_option('hooks')) - run_hook_kwargs = { - 'interactive': True, - 'work_dir': os.getcwd(), - } - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) - if isinstance(hook_res, str): - cmd, old_cmd = hook_res, cmd - _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", - RUN_SHELL_CMD, cmd, old_cmd) - - # # Log command output - if cmd_log: - cmd_log.write("# output for interactive command: %s\n\n" % cmd) - - # Make sure we close the proc handles and the cmd_log file - @contextlib.contextmanager - def get_proc(): - try: - proc = asyncprocess.Popen(cmd, shell=True, stdout=asyncprocess.PIPE, stderr=asyncprocess.STDOUT, - stdin=asyncprocess.PIPE, close_fds=True, executable='/bin/bash') - except OSError as err: - if cmd_log: - cmd_log.close() - raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err) - try: - yield proc - finally: - if proc.stdout: - proc.stdout.close() - if proc.stdin: - proc.stdin.close() - if cmd_log: - cmd_log.close() - - with get_proc() as proc: - ec = proc.poll() - stdout_err = '' - old_len_out = -1 - hit_count = 0 - - while ec is None: - # need to read from time to time. - # - otherwise the stdout/stderr buffer gets filled and it all stops working - try: - out = get_output_from_process(proc, asynchronous=True) - - if cmd_log: - cmd_log.write(out) - stdout_err += out - # recv_some used by get_output_from_process for getting asynchronous output may throw exception - except (IOError, Exception) as err: - _log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err) - out = None - - hit = False - for question, answers in new_qa.items(): - res = question.search(stdout_err) - if out and res: - fa = answers[0] % res.groupdict() - # cycle through list of answers - last_answer = answers.pop(0) - answers.append(last_answer) - _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers) - - _log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:]) - asyncprocess.send_all(proc, fa) - hit = True - break - if not hit: - for question, answers in new_std_qa.items(): - res = question.search(stdout_err) - if out and res: - fa = answers[0] % res.groupdict() - # cycle through list of answers - last_answer = answers.pop(0) - answers.append(last_answer) - _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers) - - _log.debug("run_cmd_qa answer %s std question %s out %s", - fa, question.pattern, stdout_err[-50:]) - asyncprocess.send_all(proc, fa) - hit = True - break - if not hit: - if len(stdout_err) > old_len_out: - old_len_out = len(stdout_err) - else: - noqa = False - for r in new_no_qa: - if r.search(stdout_err): - _log.debug("runqanda: noQandA found for out %s", stdout_err[-50:]) - noqa = True - if not noqa: - hit_count += 1 - else: - hit_count = 0 - else: - hit_count = 0 - - if hit_count > maxhits: - # explicitly kill the child process before exiting - try: - os.killpg(proc.pid, signal.SIGKILL) - os.kill(proc.pid, signal.SIGKILL) - except OSError as err: - _log.debug("run_cmd_qa exception caught when killing child process: %s", err) - _log.debug("run_cmd_qa: full stdouterr: %s", stdout_err) - raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s", - cmd, maxhits, stdout_err[-500:]) - - # the sleep below is required to avoid exiting on unknown 'questions' too early (see above) - time.sleep(1) - ec = proc.poll() - - # Process stopped. Read all remaining data - try: - if proc.stdout: - out = get_output_from_process(proc) - stdout_err += out - if cmd_log: - cmd_log.write(out) - except IOError as err: - _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err) - - run_hook_kwargs.update({ - 'interactive': True, - 'exit_code': ec, - 'output': stdout_err, - }) - run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) - - if trace: - trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) - - try: - os.chdir(cwd) - except OSError as err: - raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) - - return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp) - - -def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): - """ - Parse command output and construct return value. - :param cmd: executed command - :param stdouterr: combined stdout/stderr of executed command - :param ec: exit code of executed command - :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) - :param log_all: always log command output and exit code - :param log_ok: only run output/exit code for failing commands (exit code non-zero) - :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) - """ - if strictness == IGNORE: - check_ec = False - fail_on_error_match = False - elif strictness == WARN: - check_ec = True - fail_on_error_match = False - elif strictness == ERROR: - check_ec = True - fail_on_error_match = True - else: - raise EasyBuildError("invalid strictness setting: %s", strictness) - - # allow for overriding the regexp setting - if not regexp: - fail_on_error_match = False - - if ec and (log_all or log_ok): - # We don't want to error if the user doesn't care - if check_ec: - raise EasyBuildError('cmd "%s" exited with exit code %s and output:\n%s', cmd, ec, stdouterr) - else: - _log.warning('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr)) - elif not ec: - if log_all: - _log.info('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr)) - else: - _log.debug('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr)) - - # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in - if fail_on_error_match or regexp: - res = parse_log_for_error(stdouterr, regexp, stdout=False) - if res: - errors = "\n\t" + "\n\t".join([r[0] for r in res]) - error_str = "error" if len(res) == 1 else "errors" - if fail_on_error_match: - raise EasyBuildError("Found %s %s in output of %s:%s", len(res), error_str, cmd, errors) - else: - _log.warning("Found %s potential %s (some may be harmless) in output of %s:%s", - len(res), error_str, cmd, errors) - - if simple: - if ec: - # If the user does not care -> will return true - return not check_ec - else: - return True - else: - # Because we are not running in simple mode, we return the output and ec to the user - return (stdouterr, ec) - - -def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): - """ - txt is multiline string. - - in memory - regExp is a one-line regular expression - - default - """ - global errors_found_in_log - - if regExp and isinstance(regExp, bool): - regExp = r"(? Date: Wed, 10 Jan 2024 09:44:53 +0100 Subject: [PATCH 047/430] add params modextrapaths_append and allow_append_abs_path --- easybuild/framework/easyblock.py | 8 ++++++++ easybuild/framework/easyconfig/default.py | 2 ++ easybuild/framework/easyconfig/format/format.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9f3233c265..063b2c01cc 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1421,6 +1421,14 @@ def make_module_extra(self, altroot=None, altversion=None): value, type(value)) lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path'])) + for (key, value) in self.cfg['modextrapaths_append'].items(): + if isinstance(value, string_type): + value = [value] + elif not isinstance(value, (tuple, list)): + raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple", + value, type(value)) + lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path'])) + modloadmsg = self.cfg['modloadmsg'] if modloadmsg: # add trailing newline to prevent that shell prompt is 'glued' to module load message diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dd91229d1e..dae27aca65 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -189,10 +189,12 @@ 'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS], # MODULES easyconfig parameters + 'allow_append_abs_path': [False, "Allow specifying absolute paths to append in modextrapaths_append", MODULES], 'allow_prepend_abs_path': [False, "Allow specifying absolute paths to prepend in modextrapaths", MODULES], 'include_modpath_extensions': [True, "Include $MODULEPATH extensions specified by module naming scheme.", MODULES], 'modaliases': [{}, "Aliases to be defined in module file", MODULES], 'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES], + 'modextrapaths_append': [{}, "Extra paths to be appended in module file", MODULES], 'modextravars': [{}, "Extra environment variables to be added to module file", MODULES], 'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES], 'modunloadmsg': [{}, "Message that should be printed when generated module is unloaded", MODULES], diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 9a1626c60b..d503b6703a 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -73,7 +73,7 @@ ] LAST_PARAMS = ['exts_default_options', 'exts_list', 'sanity_check_paths', 'sanity_check_commands', - 'modextrapaths', 'modextravars', + 'modextrapaths', 'modextrapaths_append', 'modextravars', 'moduleclass'] SANITY_CHECK_PATHS_DIRS = 'dirs' From 5b0d6a302501088c700fd6255a1a42bf0b7f40d4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Jan 2024 13:24:49 +0100 Subject: [PATCH 048/430] Improve --check-github output Fix misdetecting an empty username as valid. Show more information on why/what failed, e.g. for creating gists or not/partially populated --git_working_dirs_path --- easybuild/tools/github.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ca013743d7..8100c8613b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2008,17 +2008,18 @@ def check_github(): github_user = build_option('github_user') github_account = build_option('github_org') or build_option('github_user') - if github_user is None: - check_res = "(none available) => FAIL" - status['--new-pr'] = status['--update-pr'] = status['--upload-test-report'] = False - else: + if github_user: check_res = "%s => OK" % github_user + else: + check_res = "%s => FAIL" % ('(none available)' if github_user is None else '(empty)') + status['--new-pr'] = status['--update-pr'] = status['--upload-test-report'] = False print_msg(check_res, log=_log, prefix=False) # check GitHub token print_msg("* GitHub token...", log=_log, prefix=False, newline=False) github_token = fetch_github_token(github_user) + github_token_valid = False if github_token is None: check_res = "(no token found) => FAIL" else: @@ -2027,6 +2028,7 @@ def check_github(): token_descr = partial_token + " (len: %d)" % len(github_token) if validate_github_token(github_token, github_user): check_res = "%s => OK (validated)" % token_descr + github_token_valid = True else: check_res = "%s => FAIL (validation failed)" % token_descr @@ -2119,7 +2121,7 @@ def check_github(): try: getattr(git_repo.remotes, remote_name).push(branch_name, delete=True) except GitCommandError as err: - sys.stderr.write("WARNING: failed to delete test branch from GitHub: %s\n" % err) + print_warning("failed to delete test branch from GitHub: %s" % err, log=_log) # test creating a gist print_msg("* creating gists...", log=_log, prefix=False, newline=False) @@ -2137,17 +2139,33 @@ def check_github(): if gist_url and re.match('https://gist.github.com/%s/[0-9a-f]+$' % github_user, gist_url): check_res = "OK" - else: + elif not github_user: + check_res = "FAIL (no GitHub user specified)" + elif not github_token: + check_res = "FAIL (missing github token)" + elif not github_token_valid: + check_res = "FAIL (invalid github token)" + elif gist_url: check_res = "FAIL (gist_url: %s)" % gist_url - status['--upload-test-report'] = False + else: + check_res = "FAIL" + if 'FAIL' in check_res: + status['--upload-test-report'] = False print_msg(check_res, log=_log, prefix=False) # check whether location to local working directories for Git repositories is available (not strictly needed) print_msg("* location to Git working dirs... ", log=_log, prefix=False, newline=False) git_working_dirs_path = build_option('git_working_dirs_path') if git_working_dirs_path: - check_res = "OK (%s)" % git_working_dirs_path + repos = [GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_FRAMEWORK_REPO] + missing_repos = [repo for repo in repos if not os.path.exists(os.path.join(git_working_dirs_path, repo))] + if not missing_repos: + check_res = "OK (%s)" % git_working_dirs_path + elif missing_repos != repos: + check_res = "OK (%s) but missing %s (suboptimal)" % (git_working_dirs_path, ', '.join(missing_repos)) + else: + check_res = "set (%s) but not populated (suboptimal)" % git_working_dirs_path else: check_res = "not found (suboptimal)" From 8127a99c775061f85858fa4b3646cd810df33b00 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 10:03:47 +0100 Subject: [PATCH 049/430] add tests --- test/framework/easyblock.py | 19 +++++++++++++++++++ test/framework/easyconfig.py | 2 ++ test/framework/toy_build.py | 9 +++++++++ 3 files changed, 30 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 5e2407575d..df40f42d98 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -645,6 +645,7 @@ def test_make_module_extra(self): # also check how absolute paths specified in modexself.contents = '\n'.join([ self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -657,6 +658,7 @@ def test_make_module_extra(self): # allow use of absolute paths, and verify contents of module self.contents += "\nallow_prepend_abs_path = True" + self.contents += "\nallow_append_abs_path = True" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -675,6 +677,9 @@ def test_make_module_extra(self): r"^prepend[-_]path.*TEST_PATH_VAR.*root.*foo", r"^prepend[-_]path.*TEST_PATH_VAR.*/test/absolute/path", r"^prepend[-_]path.*TEST_PATH_VAR.*root.*bar", + r"^append[-_]path.*TEST_PATH_VAR.*root.*foo", + r"^append[-_]path.*TEST_PATH_VAR.*/test/absolute/path", + r"^append[-_]path.*TEST_PATH_VAR.*root.*bar", ] for pattern in patterns: self.assertTrue(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt)) @@ -1173,6 +1178,7 @@ def test_make_module_step(self): 'PATH': ('xbin', 'pibin'), 'CPATH': 'pi/include', } + modextrapaths_append = modextrapaths.copy() self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, @@ -1186,6 +1192,7 @@ def test_make_module_step(self): "hiddendependencies = [('test', '1.2.3'), ('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]", "modextravars = %s" % str(modextravars), "modextrapaths = %s" % str(modextrapaths), + "modextrapaths_append = %s" % str(modextrapaths_append), ]) # test if module is generated correctly @@ -1256,6 +1263,18 @@ def test_make_module_step(self): num_prepends = len(regex.findall(txt)) self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) + for (key, vals) in modextrapaths_append.items(): + if isinstance(vals, string_type): + vals = [vals] + for val in vals: + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^append-path\s+%s\s+\$root/%s$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^append_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) + else: + self.fail("Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + for (name, ver) in [('GCC', '6.4.0-2.28')]: if get_module_syntax() == 'Tcl': regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index dda5c0a05b..905cf6c15b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1151,6 +1151,7 @@ def test_templating_constants(self): 'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s', ]), 'modextrapaths = {"PI_MOD_NAME": "%%(module_name)s"}', + 'modextrapaths_append = {"PATH_APPEND": "appended_path"}', 'license_file = HOME + "/licenses/PI/license.txt"', "github_account = 'easybuilders'", ]) % inp @@ -1191,6 +1192,7 @@ def test_templating_constants(self): self.assertEqual(ec['modloadmsg'], expected) self.assertEqual(ec['modunloadmsg'], expected) self.assertEqual(ec['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'}) + self.assertEqual(ec['modextrapaths_append'], {'PATH_APPEND': 'appended_path'}) self.assertEqual(ec['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) # test the escaping insanity here (ie all the crap we allow in easyconfigs) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index cc84ce86d1..e5469dd488 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -285,6 +285,7 @@ def test_toy_tweaked(self): ec_extra = '\n'.join([ "versionsuffix = '-tweaked'", "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}", + "modextrapaths_append = {'SOMEPATH_APPEND': ['qux/fred', 'thud', '']}", "modextravars = {'FOO': 'bar'}", "modloadmsg = '%s'" % modloadmsg, "modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua @@ -319,6 +320,9 @@ def test_toy_tweaked(self): self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/baz$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root/qux/fred$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root/thud$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root$', toy_module_txt, re.M)) mod_load_msg = r'module-info mode load.*\n\s*puts stderr\s*.*%s$' % modloadmsg_regex_tcl self.assertTrue(re.search(mod_load_msg, toy_module_txt, re.M)) self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M)) @@ -326,6 +330,11 @@ def test_toy_tweaked(self): self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M)) pattern = r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$' self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + pattern = r'^append_path\("SOMEPATH_APPEND", pathJoin\(root, "qux/fred"\)\)$' + self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + pattern = r'^append_path\("SOMEPATH_APPEND", pathJoin\(root, "thud"\)\)$' + self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append_path\("SOMEPATH_APPEND", root\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M)) mod_load_msg = r'^if mode\(\) == "load" then\n\s*io.stderr:write\(%s\)$' % modloadmsg_regex_lua From 4e0a068e79d223390cd5fec443cc9cc7d0b5c91b Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 10:20:57 +0100 Subject: [PATCH 050/430] fix tests --- test/framework/toy_build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index e5469dd488..0349652b47 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1523,6 +1523,9 @@ def test_toy_module_fulltxt(self): r'prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)', r'prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)', r'prepend_path\("SOMEPATH", root\)', + r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "qux/fred"\)\)', + r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "thud"\)\)', + r'append_path\("SOMEPATH_APPEND", root\)', r'', r'if mode\(\) == "load" then', ] + modloadmsg_lua + [ @@ -1561,6 +1564,9 @@ def test_toy_module_fulltxt(self): r'prepend-path SOMEPATH \$root/foo/bar', r'prepend-path SOMEPATH \$root/baz', r'prepend-path SOMEPATH \$root', + r'append-path SOMEPATH_APPEND \$root/qux/fred', + r'append-path SOMEPATH_APPEND \$root/thud', + r'append-path SOMEPATH_APPEND \$root', r'', r'if { \[ module-info mode load \] } {', ] + modloadmsg_tcl + [ From 2eef785818183d8fd3d53423d0e723729c810460 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 13:13:41 +0100 Subject: [PATCH 051/430] try to fix tests --- test/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index df40f42d98..27ece36ca5 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -644,8 +644,8 @@ def test_make_module_extra(self): installver = '3.14-gompi-2018a' # also check how absolute paths specified in modexself.contents = '\n'.join([ - self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" - self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -677,9 +677,9 @@ def test_make_module_extra(self): r"^prepend[-_]path.*TEST_PATH_VAR.*root.*foo", r"^prepend[-_]path.*TEST_PATH_VAR.*/test/absolute/path", r"^prepend[-_]path.*TEST_PATH_VAR.*root.*bar", - r"^append[-_]path.*TEST_PATH_VAR.*root.*foo", - r"^append[-_]path.*TEST_PATH_VAR.*/test/absolute/path", - r"^append[-_]path.*TEST_PATH_VAR.*root.*bar", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*foo", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*/test/absolute/path", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*bar", ] for pattern in patterns: self.assertTrue(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt)) @@ -1178,7 +1178,7 @@ def test_make_module_step(self): 'PATH': ('xbin', 'pibin'), 'CPATH': 'pi/include', } - modextrapaths_append = modextrapaths.copy() + modextrapaths_append = {'APPEND_PATH': 'append_path'} self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, From 58d67dbed338aa42ca5fa2076cfe5f0a96eccb21 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 15:56:09 +0100 Subject: [PATCH 052/430] fix --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 27ece36ca5..6c08f947d9 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -644,7 +644,7 @@ def test_make_module_extra(self): installver = '3.14-gompi-2018a' # also check how absolute paths specified in modexself.contents = '\n'.join([ - self.contents += "\nmodextrapaths = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) From 571b94dec403a7afd8e4463fe6b1cd8cc59de373 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 15 Jan 2024 15:36:32 +0100 Subject: [PATCH 053/430] Move post install commands after lib64 symlinking --- easybuild/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0f1c6d6f3f..c8967e812c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3019,12 +3019,6 @@ def post_install_step(self): - run post install commands if any were specified """ - self.run_post_install_commands() - self.apply_post_install_patches() - self.print_post_install_messages() - - self.fix_shebang() - lib_dir = os.path.join(self.installdir, 'lib') lib64_dir = os.path.join(self.installdir, 'lib64') @@ -3045,6 +3039,12 @@ def post_install_step(self): # create *relative* 'lib' symlink to 'lib64'; symlink('lib64', lib_dir, use_abspath_source=False) + self.run_post_install_commands() + self.apply_post_install_patches() + self.print_post_install_messages() + + self.fix_shebang() + def sanity_check_step(self, *args, **kwargs): """ Do a sanity check on the installation From b03f26e9ebaa9de6407b1e40459c1d586c3c97d1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 15 Jan 2024 16:44:10 +0100 Subject: [PATCH 054/430] Add script for updating local git repos with develop branch Usefull when using `--git-working-dirs-path` and/or with checkouts created by `install-EasyBuild-develop.sh`. --- .../scripts/install-EasyBuild-develop.sh | 9 +- easybuild/scripts/update-EasyBuild-develop.sh | 91 +++++++++++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) create mode 100755 easybuild/scripts/update-EasyBuild-develop.sh diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh index 4181d8c42a..f88ae280a0 100755 --- a/easybuild/scripts/install-EasyBuild-develop.sh +++ b/easybuild/scripts/install-EasyBuild-develop.sh @@ -14,7 +14,7 @@ print_usage() echo echo " github_username: username on GitHub for which the EasyBuild repositories should be cloned" echo - echo " install_dir: directory were all the EasyBuild files will be installed" + echo " install_dir: directory where all the EasyBuild files will be installed" echo } @@ -79,7 +79,7 @@ EOF # Check for 'help' argument -if [ "$1" = "-h" -o "$1" = "--help" ] ; then +if [ "$1" = "-h" ] || [ "$1" = "--help" ] ; then print_usage exit 0 fi @@ -116,13 +116,14 @@ github_clone_branch "easybuild" "develop" EB_DEVEL_MODULE_NAME="EasyBuild-develop" MODULES_INSTALL_DIR=${INSTALL_DIR}/modules EB_DEVEL_MODULE="${MODULES_INSTALL_DIR}/${EB_DEVEL_MODULE_NAME}" -mkdir -p ${MODULES_INSTALL_DIR} +mkdir -p "${MODULES_INSTALL_DIR}" print_devel_module > "${EB_DEVEL_MODULE}" -echo +echo echo "=== Run 'module use ${MODULES_INSTALL_DIR}' and 'module load ${EB_DEVEL_MODULE_NAME}' to use your development version of EasyBuild." echo "=== (you can append ${MODULES_INSTALL_DIR} to your MODULEPATH to make this module always available for loading)" echo echo "=== To update each repository, run 'git pull origin' in each subdirectory of ${INSTALL_DIR}" +echo "=== Or run $(dirname "$0")/update-EasyBuild-develop.sh '${INSTALL_DIR}'" echo exit 0 diff --git a/easybuild/scripts/update-EasyBuild-develop.sh b/easybuild/scripts/update-EasyBuild-develop.sh new file mode 100755 index 0000000000..c80289ad96 --- /dev/null +++ b/easybuild/scripts/update-EasyBuild-develop.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# Stop in case of error +set -e + +# Print script help +print_usage() +{ + echo "Checkout develop branch of all EasyBuild repositories" + echo "and pull changes from the remote repository." + echo "To be used with the EasyBuild-develop module or a set git-working-dirs-path" + echo "Usage: $0 []" + echo + echo " git_dir: directory where all the EasyBuild repositories are installed." + echo " Automatically detected if not specified." + echo +} + +if [[ "$1" = "-h" ]] || [[ "$1" = "--help" ]]; then + print_usage + exit 0 +fi + +if [[ $# -gt 1 ]] ; then + echo "Error: invalid arguments" + echo + print_usage + exit 1 +fi + +if [[ $# -eq 1 ]]; then + git_dir=$1 +else + # Auto detect git_dir + git_dir="" + if ! which eb &> /dev/null; then + module load EasyBuild-develop || module load EasyBuild || true + if ! which eb &> /dev/null; then + echo 'Found neither the `eb` command nor a working module.' + echo 'Please specify the git_dir!' + exit 1 + fi + fi + if out=$(eb --show-config | grep -F 'git-working-dirs-path'); then + path=$(echo "$out" | awk '{print $NF}') + if [[ -n "$path" ]] && [[ -d "$path" ]]; then + git_dir=$path + echo "Using git_dir from git-working-dirs-path: $git_dir" + fi + fi + if [[ -z "$git_dir" ]]; then + eb_dir=$(dirname "$(which eb)") + if [[ "$(basename "$eb_dir")" == "easybuild-framework" ]] && [[ -d "$eb_dir/.git" ]]; then + git_dir=$(dirname "$eb_dir") + echo "Using git_dir from eb command: $git_dir" + else + echo 'Please specify the git_dir as auto-detection failed!' + exit 1 + fi + fi +fi + +cd "$git_dir" + +for folder in easybuild easybuild-framework easybuild-easyblocks easybuild-easyconfigs; do + echo # A newline + if [[ -d "$folder" ]]; then + echo "========= Checking ${folder} =========" + else + echo "========= Skipping non-existent ${folder} =========" + fi + cd "$folder" + git checkout "develop" + if git remote | grep -qF github_easybuilders; then + git pull "github_easybuilders" + else + git pull + fi + cd .. +done + +index_file="$git_dir/easybuild-easyconfigs/easybuild/easyconfigs/.eb-path-index" +if [[ -f "$index_file" ]]; then + echo -n "Trying to remove index from ${index_file}..." + if rm "$index_file"; then + echo "Done!" + echo "Recreate with 'eb --create-index \"$(dirname "$index_file")\"'" + else + echo "Failed!" + fi +fi From edeb30a194b09131a0d8785dff77a4547ff601ee Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:05:36 +0000 Subject: [PATCH 055/430] Deprecate EnvironmentModulesC and EnvironmentModulesTcl --- easybuild/tools/modules.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index c0d8c25c59..bdf764e9cc 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1185,6 +1185,7 @@ class EnvironmentModulesC(ModulesTool): COMMAND = "modulecmd" REQ_VERSION = '3.2.10' MAX_VERSION = '3.99' + DEPR_VERSION = '3.999' VERSION_REGEXP = r'^\s*(VERSION\s*=\s*)?(?P\d\S*)\s*' def run_module(self, *args, **kwargs): @@ -1246,6 +1247,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC): COMMAND_SHELL = ['tclsh'] VERSION_OPTION = '' REQ_VERSION = None + DEPR_VERSION = '9999' VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' def set_path_env_var(self, key, paths): From 289ff54b64b90afce798a9d8bdf34b75251f1009 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:12:18 +0000 Subject: [PATCH 056/430] update deprecation message --- easybuild/tools/modules.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index bdf764e9cc..2df2e165e9 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -272,11 +272,7 @@ def set_and_check_version(self): if StrictVersion(self.version) < StrictVersion(self.DEPR_VERSION): depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version - - if self.version.startswith('6') and 'Lmod6' in build_option('silence_deprecation_warnings'): - self.log.warning(depr_msg) - else: - self.log.deprecated(depr_msg, '5.0') + self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) From bd99f6578967e28a8a1cbc99d1f9f5029cd054cd Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 16 Jan 2024 10:14:03 +0100 Subject: [PATCH 057/430] Error when multiple PR options are passed Passing `--preview-pr` in addition to `--new-pr` will not show a preview but actually create a new PR. In general it is always wrong to pass multiple PR-options to EB. So check for that and error out with an appropriate error message. --- easybuild/main.py | 33 +++++++++++++++++++++++---------- test/framework/options.py | 10 ++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 73550e3998..802009166e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -333,11 +333,23 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session categorized_paths = categorize_files_by_type(eb_args) + pr_options = [ + 'new_branch_github', + 'new_pr', + 'new_pr_from_branch', + 'preview_pr', + 'sync_branch_with_develop', + 'sync_pr_with_develop', + 'update_branch_github', + 'update_pr', + ] + set_pr_options = [opt for opt in pr_options if getattr(options, opt)] + any_pr_option_set = len(set_pr_options) > 0 + if len(set_pr_options) > 1: + raise EasyBuildError("The following options are set but incompatible: %s.\nRemove at least one!", + ', '.join(['--' + opt.replace('_', '-') for opt in set_pr_options])) # command line options that do not require any easyconfigs to be specified - pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr - pr_options = pr_options or options.sync_branch_with_develop or options.sync_pr_with_develop - pr_options = pr_options or options.update_branch_github or options.update_pr - no_ec_opts = [options.aggregate_regtest, options.regtest, pr_options, search_query] + no_ec_opts = [options.aggregate_regtest, options.regtest, any_pr_option_set, search_query] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) @@ -427,9 +439,10 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules - keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options or options.copy_ec - keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only - keep_available_modules = keep_available_modules or options.inject_checksums_to_json + keep_available_modules = any(( + forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums, + options.sanity_check_only, options.inject_checksums_to_json) + ) # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not keep_available_modules: @@ -448,12 +461,12 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled - if options.robot and (not dry_run_mode or pr_options): + if options.robot and (not dry_run_mode or any_pr_option_set): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs - elif pr_options: + elif any_pr_option_set: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) @@ -472,7 +485,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return True # creating/updating PRs - if pr_options: + if any_pr_option_set: if options.new_pr: new_pr(categorized_paths, ordered_ecs) elif options.new_branch_github: diff --git a/test/framework/options.py b/test/framework/options.py index 4b5e1afce8..0067525bde 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3915,6 +3915,16 @@ def test_github_review_pr(self): self.mock_stderr(False) self.assertNotIn("2016.04", txt) + def test_set_multiple_pr_opts(self): + """Test that passing multiple PR options results in an error""" + test_cases = [ + ['--new-pr', 'dummy.eb', '--preview-pr'], + ['--new-pr', 'dummy.eb', '--update-pr', '42'], + ] + for args in test_cases: + error_pattern = "The following options are set but incompatible.* " + args[0] + self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True) + def test_set_tmpdir(self): """Test set_tmpdir config function.""" self.purge_environment() From 1da0e55c256d63fb75fce06772b39cff270e54cf Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 16 Jan 2024 10:25:08 +0100 Subject: [PATCH 058/430] Remove pr_options variable Use a tuple in the list generator to avoid accidentally using that variable which now contained ALL options not if any one was set (as before). --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 802009166e..90ee5882fc 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -333,7 +333,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session categorized_paths = categorize_files_by_type(eb_args) - pr_options = [ + set_pr_options = [opt for opt in ( 'new_branch_github', 'new_pr', 'new_pr_from_branch', @@ -342,8 +342,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session 'sync_pr_with_develop', 'update_branch_github', 'update_pr', + ) if getattr(options, opt) ] - set_pr_options = [opt for opt in pr_options if getattr(options, opt)] any_pr_option_set = len(set_pr_options) > 0 if len(set_pr_options) > 1: raise EasyBuildError("The following options are set but incompatible: %s.\nRemove at least one!", From a7961568c82c945f4546aa07d88e9841378fb819 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Jan 2024 12:56:33 +0100 Subject: [PATCH 059/430] don't hardcode /bin/bash in eb script, RPATH wrapper script, and run_shell_cmd --- easybuild/scripts/rpath_wrapper_template.sh.in | 2 +- easybuild/tools/run.py | 5 ++++- eb | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 55c3388c5b..73eb21f0e0 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## # Copyright 2016-2023 Ghent University # diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 0bb0d3c9f4..92c92a1b41 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -41,6 +41,7 @@ import os import re import signal +import shutil import subprocess import sys import tempfile @@ -276,7 +277,9 @@ def to_cmd_str(cmd): # (which could be dash instead of bash, like on Ubuntu, see https://wiki.ubuntu.com/DashAsBinSh) # stick to None (default value) when not running command via a shell if use_bash: - executable, shell = '/bin/bash', True + bash = shutil.which('bash') + _log.info(f"Path to bash that will be used to run shell commands: {bash}") + executable, shell = bash, True else: executable, shell = None, False diff --git a/eb b/eb index 402bb87d0a..f427c5e7e7 100755 --- a/eb +++ b/eb @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## # Copyright 2009-2023 Ghent University # From 0b68c30ab63f3ce8ed21adb3eb33105c60b2adf2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 17 Jan 2024 14:06:53 +0100 Subject: [PATCH 060/430] fix broken test_extensions_sanity_check which was hardcoding /bin/bash in expected error message --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 9f656573d1..40ed5f692a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2082,7 +2082,7 @@ def test_extensions_sanity_check(self): eb.silent = True error_pattern = r"Sanity check failed: extensions sanity check failed for 1 extensions: toy\n" error_pattern += r"failing sanity check for 'toy' extension: " - error_pattern += r'command "thisshouldfail" failed; output:\n/bin/bash:.* thisshouldfail: command not found' + error_pattern += r'command "thisshouldfail" failed; output:\n.* thisshouldfail: command not found' with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, eb.run_all_steps, True) From a468ef9f3eb083da8e97285f720bc5e58f28f061 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Jan 2024 17:32:02 +0100 Subject: [PATCH 061/430] Enhance error message --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 90ee5882fc..85501f57f0 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -346,7 +346,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session ] any_pr_option_set = len(set_pr_options) > 0 if len(set_pr_options) > 1: - raise EasyBuildError("The following options are set but incompatible: %s.\nRemove at least one!", + raise EasyBuildError("The following options are set but incompatible: %s.\nYou can only use one at a time!", ', '.join(['--' + opt.replace('_', '-') for opt in set_pr_options])) # command line options that do not require any easyconfigs to be specified no_ec_opts = [options.aggregate_regtest, options.regtest, any_pr_option_set, search_query] From 17bac3147b220b586b2dadd0b0cb4817f2e01c8e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Jan 2024 17:34:11 +0100 Subject: [PATCH 062/430] Add more test cases --- test/framework/options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 0067525bde..f81d34fff8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3920,6 +3920,8 @@ def test_set_multiple_pr_opts(self): test_cases = [ ['--new-pr', 'dummy.eb', '--preview-pr'], ['--new-pr', 'dummy.eb', '--update-pr', '42'], + ['--new-pr', 'dummy.eb', '--sync-pr-with-develop', '42'], + ['--new-pr', 'dummy.eb', '--new-pr-from-branch', 'mybranch'], ] for args in test_cases: error_pattern = "The following options are set but incompatible.* " + args[0] From 8aaaec2904ca2a260e0d4501f6d8da5b553f4af9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 19:13:25 +0100 Subject: [PATCH 063/430] replace string_type with str in easyblock.py --- easybuild/framework/easyblock.py | 2 +- test/framework/easyblock.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index edf4b01f8a..f6077dd5ce 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1413,7 +1413,7 @@ def make_module_extra(self, altroot=None, altversion=None): lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path'])) for (key, value) in self.cfg['modextrapaths_append'].items(): - if isinstance(value, string_type): + if isinstance(value, str): value = [value] elif not isinstance(value, (tuple, list)): raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple", diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 3eed2986b2..64c554ca89 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1273,7 +1273,7 @@ def test_make_module_step(self): self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) for (key, vals) in modextrapaths_append.items(): - if isinstance(vals, string_type): + if isinstance(vals, str): vals = [vals] for val in vals: if get_module_syntax() == 'Tcl': From 7cba1dcf64fdb750a95bc0371a72300826014c1c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 21:40:28 +0100 Subject: [PATCH 064/430] implement initial support for running shell commands asynchronously using run_shell_cmd --- easybuild/tools/run.py | 40 +++++++++------ test/framework/run.py | 95 +++++++++++++++++++++++++++++------ test/framework/systemtools.py | 4 +- 3 files changed, 107 insertions(+), 32 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 92c92a1b41..c8d0cabf71 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -45,6 +45,7 @@ import subprocess import sys import tempfile +import threading import time from collections import namedtuple from datetime import datetime @@ -79,7 +80,7 @@ RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', - 'out_file', 'err_file')) + 'out_file', 'err_file', 'thread_id')) class RunShellCmdError(BaseException): @@ -199,7 +200,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param use_bash: execute command through bash shell (enabled by default) :param output_file: collect command output in temporary output file :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None) - :param asynchronous: run command asynchronously + :param asynchronous: indicate that command is being run asynchronously :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions @@ -223,9 +224,6 @@ def to_cmd_str(cmd): return cmd_str # temporarily raise a NotImplementedError until all options are implemented - if asynchronous: - raise NotImplementedError - if qa_patterns or qa_wait_patterns: raise NotImplementedError @@ -235,6 +233,11 @@ def to_cmd_str(cmd): cmd_str = to_cmd_str(cmd) cmd_name = os.path.basename(cmd_str.split(' ')[0]) + thread_id = None + if asynchronous: + thread_id = threading.get_native_id() + _log.info(f"Initiating running of shell command '{cmd_str}' via thread with ID {thread_id}") + # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): _log.info(f"Auto-enabling streaming output of '{cmd_str}' command because logging to stdout is enabled") @@ -259,16 +262,16 @@ def to_cmd_str(cmd): if not in_dry_run and build_option('extended_dry_run'): if not hidden or verbose_dry_run: silent = build_option('silent') - msg = f" running command \"{cmd_str}\"\n" + msg = f" running shell command \"{cmd_str}\"\n" msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) start_time = datetime.now() if not hidden: - cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp) + _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id) if stream_output: print_msg(f"(streaming) output for command '{cmd_str}':") @@ -293,7 +296,11 @@ def to_cmd_str(cmd): stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT - _log.info(f"Running command '{cmd_str}' in {work_dir}") + log_msg = f"Running shell command '{cmd_str}' in {work_dir}" + if thread_id: + log_msg += f" (via thread with ID {thread_id})" + _log.info(log_msg) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, stdin=subprocess.PIPE, cwd=work_dir, env=env, shell=shell, executable=executable) @@ -337,7 +344,7 @@ def to_cmd_str(cmd): raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) # always log command output cmd_name = cmd_str.split(' ')[0] @@ -370,7 +377,7 @@ def to_cmd_str(cmd): return res -def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): +def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id): """ Helper function to construct and print trace message for command being run @@ -380,11 +387,18 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): :param stdin: stdin input value for command :param cmd_out_fp: path to output file for command :param cmd_err_fp: path to errors/warnings output file for command + :param thread_id: thread ID (None when not running shell command asynchronously) """ start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') + if thread_id: + run_cmd_msg = f"running shell command (asynchronously, thread ID: {thread_id}):" + else: + run_cmd_msg = "running shell command:" + lines = [ - "running command:", + run_cmd_msg, + f"\t{cmd}", f"\t[started at: {start_time}]", f"\t[working dir: {work_dir}]", ] @@ -395,8 +409,6 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): if cmd_err_fp: lines.append(f"\t[errors/warnings saved to {cmd_err_fp}]") - lines.append('\t' + cmd) - trace_msg('\n'.join(lines)) diff --git a/test/framework/run.py b/test/framework/run.py index db74940aec..9a9b02d767 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -41,6 +41,7 @@ import tempfile import textwrap import time +from concurrent.futures import ThreadPoolExecutor from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from easybuild.base.fancylogger import setLogLevelDebug @@ -248,7 +249,7 @@ def test_run_shell_cmd_log(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) - regex_start_cmd = re.compile("Running command 'echo hello' in /") + regex_start_cmd = re.compile("Running shell command 'echo hello' in /") regex_cmd_exit = re.compile(r"Shell command completed successfully \(see output above\): echo hello") # command output is always logged @@ -448,7 +449,7 @@ def test_run_cmd_work_dir(self): def test_run_shell_cmd_work_dir(self): """ - Test running command in specific directory with run_shell_cmd function. + Test running shell command in specific directory with run_shell_cmd function. """ orig_wd = os.getcwd() self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) @@ -615,11 +616,11 @@ def test_run_shell_cmd_trace(self): """Test run_shell_cmd function in trace mode, and with tracing disabled.""" pattern = [ - r"^ >> running command:", + r"^ >> running shell command:", + r"\techo hello", r"\t\[started at: .*\]", r"\t\[working dir: .*\]", r"\t\[output saved to .*\]", - r"\techo hello", r" >> command completed: exit 0, ran in .*", ] @@ -675,11 +676,11 @@ def test_run_shell_cmd_trace_stdin(self): init_config(build_options={'trace': True}) pattern = [ - r"^ >> running command:", + r"^ >> running shell command:", + r"\techo hello", r"\t\[started at: [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\]", r"\t\[working dir: .*\]", r"\t\[output saved to .*\]", - r"\techo hello", r" >> command completed: exit 0, ran in .*", ] @@ -707,8 +708,8 @@ def test_run_shell_cmd_trace_stdin(self): self.assertEqual(res.output, 'hello') self.assertEqual(res.exit_code, 0) self.assertEqual(stderr, '') - pattern.insert(3, r"\t\[input: hello\]") - pattern[-2] = "\tcat" + pattern.insert(4, r"\t\[input: hello\]") + pattern[1] = "\tcat" regex = re.compile('\n'.join(pattern)) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) @@ -909,7 +910,8 @@ def test_run_shell_cmd_cache(self): # inject value into cache to check whether executing command again really returns cached value with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, - work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None) + work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None, + thread_id=None) run_shell_cmd.update_cache({(cmd, None): cached_res}) res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) @@ -928,7 +930,8 @@ def test_run_shell_cmd_cache(self): # inject different output for cat with 'foo' as stdin to check whether cached value is used with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, - work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None) + work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None, + thread_id=None) run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) @@ -1006,7 +1009,7 @@ def test_run_shell_cmd_dry_run(self): self.assertEqual(res.output, '') self.assertEqual(res.stderr, None) # check dry run output - expected = """ running command "somecommand foo 123 bar"\n""" + expected = """ running shell command "somecommand foo 123 bar"\n""" self.assertIn(expected, stdout) # check enabling 'hidden' @@ -1029,7 +1032,7 @@ def test_run_shell_cmd_dry_run(self): fail_on_error=False, in_dry_run=True) stdout = self.get_stdout() self.mock_stdout(False) - self.assertNotIn('running command "', stdout) + self.assertNotIn('running shell command "', stdout) self.assertNotEqual(res.exit_code, 0) self.assertEqual(res.output, 'done\n') self.assertEqual(res.stderr, None) @@ -1207,7 +1210,7 @@ def test_run_cmd_async(self): "for i in $(seq 1 50)", "do sleep 0.1", "for j in $(seq 1000)", - "do echo foo", + "do echo foo${i}${j}", "done", "done", "echo done", @@ -1257,8 +1260,68 @@ def test_run_cmd_async(self): res = check_async_cmd(*cmd_info, output=res['output']) self.assertEqual(res['done'], True) self.assertEqual(res['exit_code'], 0) - self.assertTrue(res['output'].startswith('start\n')) - self.assertTrue(res['output'].endswith('\ndone\n')) + self.assertEqual(len(res['output']), 435661) + self.assertTrue(res['output'].startswith('start\nfoo11\nfoo12\n')) + self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res['output']) + self.assertTrue(res['output'].endswith('\nfoo501000\ndone\n')) + + def test_run_shell_cmd_async(self): + """Test asynchronously running of a shell command via run_shell_cmd """ + + thread_pool = ThreadPoolExecutor() + + os.environ['TEST'] = 'test123' + env = os.environ.copy() + + test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" + task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) + + # change value of $TEST to check that command is completed with correct environment + os.environ['TEST'] = 'some_other_value' + + # initial poll should result in None, since it takes a while for the command to complete + self.assertEqual(task.done(), False) + + # wait until command is done + while not task.done(): + time.sleep(1) + res = task.result() + + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, 'sleeping...\ntest123\n') + + # check asynchronous running of failing command + error_test_cmd = "echo 'FAIL!' >&2; exit 123" + task = thread_pool.submit(run_shell_cmd, error_test_cmd, hidden=True, fail_on_error=False, asynchronous=True) + time.sleep(1) + res = task.result() + self.assertEqual(res.exit_code, 123) + self.assertEqual(res.output, "FAIL!\n") + self.assertTrue(res.thread_id) + + # also test with a command that produces a lot of output, + # since that tends to lock up things unless we frequently grab some output... + verbose_test_cmd = ';'.join([ + "echo start", + "for i in $(seq 1 50)", + "do sleep 0.1", + "for j in $(seq 1000)", + "do echo foo${i}${j}", + "done", + "done", + "echo done", + ]) + task = thread_pool.submit(run_shell_cmd, verbose_test_cmd, hidden=True, asynchronous=True) + + while not task.done(): + time.sleep(1) + res = task.result() + + self.assertEqual(res.exit_code, 0) + self.assertEqual(len(res.output), 435661) + self.assertTrue(res.output.startswith('start\nfoo11\nfoo12\n')) + self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res.output) + self.assertTrue(res.output.endswith('\nfoo501000\ndone\n')) def test_check_log_for_errors(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') @@ -1373,7 +1436,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): def test_run_shell_cmd_with_hooks(self): """ - Test running command with run_shell_cmd function with pre/post run_shell_cmd hooks in place. + Test running shell command with run_shell_cmd function with pre/post run_shell_cmd hooks in place. """ cwd = os.getcwd() diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 5a8d1033a5..5f4b62f09c 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -341,7 +341,7 @@ def mocked_run_shell_cmd(cmd, **kwargs): } if cmd in known_cmds: return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(), - out_file=None, err_file=None) + out_file=None, err_file=None, thread_id=None) else: return run_shell_cmd(cmd, **kwargs) @@ -774,7 +774,7 @@ def test_gcc_version_darwin(self): out = "Apple LLVM version 7.0.0 (clang-700.1.76)" cwd = os.getcwd() mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd, - out_file=None, err_file=None) + out_file=None, err_file=None, thread_id=None) st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) From 1c7c8b4f668dd5d9908bfb5cf92a8d88faef20f6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 22:07:41 +0100 Subject: [PATCH 065/430] use ThreadPoolExecutor to asynchronously run shell commands in EasyBlock.skip_extensions_parallel --- easybuild/framework/easyblock.py | 30 ++++++++++++++++++------------ test/framework/toy_build.py | 6 +++--- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f6077dd5ce..9744397b6b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -52,6 +52,7 @@ import tempfile import time import traceback +from concurrent.futures import ThreadPoolExecutor from datetime import datetime import easybuild.tools.environment as env @@ -87,7 +88,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import RunShellCmdError, check_async_cmd, run_cmd, run_shell_cmd +from easybuild.tools.run import RunShellCmdError, run_shell_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -1818,7 +1819,9 @@ def skip_extensions_parallel(self, exts_filter): self.log.experimental("Skipping installed extensions in parallel") print_msg("skipping installed extensions (in parallel)", log=self.log) - async_cmd_info_cache = {} + thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) + + async_shell_cmd_tasks = {} running_checks_ids = [] installed_exts_ids = [] exts_queue = list(enumerate(self.ext_instances[:])) @@ -1831,14 +1834,14 @@ def skip_extensions_parallel(self, exts_filter): # first handle completed checks for idx in running_checks_ids[:]: ext_name = self.ext_instances[idx].name - # don't read any output, just check whether command completed - async_cmd_info = check_async_cmd(*async_cmd_info_cache[idx], output_read_size=0, fail_on_error=False) - if async_cmd_info['done']: - out, ec = async_cmd_info['output'], async_cmd_info['exit_code'] - self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_name, ec, out) + # check whether command completed + task = async_shell_cmd_tasks[idx] + if task.done(): + res = task.result() + self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") running_checks_ids.remove(idx) - if ec == 0: - print_msg("skipping extension %s" % ext_name, log=self.log) + if res.exit_code == 0: + print_msg(f"skipping extension {ext_name}", log=self.log) installed_exts_ids.append(idx) checked_exts_cnt += 1 @@ -1847,11 +1850,12 @@ def skip_extensions_parallel(self, exts_filter): self.update_exts_progress_bar(exts_pbar_label) # start additional checks asynchronously - while exts_queue and len(running_checks_ids) < self.cfg['parallel']: + while exts_queue: idx, ext = exts_queue.pop(0) cmd, stdin = resolve_exts_filter_template(exts_filter, ext) - async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, - regexp=False, trace=False, asynchronous=True) + task = thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, + fail_on_error=False, asynchronous=True) + async_shell_cmd_tasks[idx] = task running_checks_ids.append(idx) # compose new list of extensions, skip over the ones that are already installed; @@ -1864,6 +1868,8 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances + thread_pool.shutdown() + def install_extensions(self, install=True): """ Install extensions. diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6ba47d3587..936fd09c4c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1917,7 +1917,7 @@ def test_toy_exts_parallel(self): write_file(test_ec, test_ec_txt) args = ['--parallel-extensions-install', '--experimental', '--force', '--parallel=3'] - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # take into account that each of these lines may appear multiple times, @@ -1936,7 +1936,7 @@ def test_toy_exts_parallel(self): # also test skipping of extensions in parallel args.append('--skip') - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # order in which these patterns occur is not fixed, so check them one by one @@ -1962,7 +1962,7 @@ def test_toy_exts_parallel(self): write_file(toy_ext_eb, toy_ext_eb_txt) args[-1] = '--include-easyblocks=%s' % toy_ext_eb - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # take into account that each of these lines may appear multiple times, # in case no progress was made between checks From e91dbfabc4a4eae9500ef4981f9671b53ab0831f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 22:34:38 +0100 Subject: [PATCH 066/430] use concurrent.futures.wait in EasyBlock.skip_extensions_parallel --- easybuild/framework/easyblock.py | 44 +++++++++++++++----------------- easybuild/tools/run.py | 8 +++--- test/framework/run.py | 4 +-- test/framework/systemtools.py | 4 +-- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9744397b6b..4f45dbbaab 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -41,7 +41,6 @@ * Davide Vanzo (Vanderbilt University) * Caspar van Leeuwen (SURF) """ - import copy import glob import inspect @@ -52,7 +51,7 @@ import tempfile import time import traceback -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from datetime import datetime import easybuild.tools.environment as env @@ -1821,42 +1820,41 @@ def skip_extensions_parallel(self, exts_filter): thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) - async_shell_cmd_tasks = {} - running_checks_ids = [] + async_shell_cmd_tasks = [] installed_exts_ids = [] exts_queue = list(enumerate(self.ext_instances[:])) checked_exts_cnt = 0 exts_cnt = len(self.ext_instances) + done_tasks = [] # asynchronously run checks to see whether extensions are already installed - while exts_queue or running_checks_ids: + while exts_queue or async_shell_cmd_tasks: # first handle completed checks - for idx in running_checks_ids[:]: + for task in done_tasks: + async_shell_cmd_tasks.remove(task) + res = task.result() + idx = res.task_id ext_name = self.ext_instances[idx].name - # check whether command completed - task = async_shell_cmd_tasks[idx] - if task.done(): - res = task.result() - self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") - running_checks_ids.remove(idx) - if res.exit_code == 0: - print_msg(f"skipping extension {ext_name}", log=self.log) - installed_exts_ids.append(idx) - - checked_exts_cnt += 1 - exts_pbar_label = "skipping installed extensions " - exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) - self.update_exts_progress_bar(exts_pbar_label) + self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") + if res.exit_code == 0: + print_msg(f"skipping extension {ext_name}", log=self.log) + installed_exts_ids.append(idx) + + checked_exts_cnt += 1 + exts_pbar_label = "skipping installed extensions " + exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) + self.update_exts_progress_bar(exts_pbar_label) # start additional checks asynchronously while exts_queue: idx, ext = exts_queue.pop(0) cmd, stdin = resolve_exts_filter_template(exts_filter, ext) task = thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, - fail_on_error=False, asynchronous=True) - async_shell_cmd_tasks[idx] = task - running_checks_ids.append(idx) + fail_on_error=False, asynchronous=True, task_id=idx) + async_shell_cmd_tasks.append(task) + + (done_tasks, _) = wait(async_shell_cmd_tasks, timeout=1, return_when=FIRST_COMPLETED) # compose new list of extensions, skip over the ones that are already installed; # note: original order in extensions list should be preserved! diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c8d0cabf71..852d13a966 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -80,7 +80,7 @@ RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', - 'out_file', 'err_file', 'thread_id')) + 'out_file', 'err_file', 'thread_id', 'task_id')) class RunShellCmdError(BaseException): @@ -184,7 +184,7 @@ def cache_aware_func(cmd, *args, **kwargs): @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, - output_file=True, stream_output=None, asynchronous=False, with_hooks=True, + output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True, qa_patterns=None, qa_wait_patterns=None): """ Run specified (interactive) shell command, and capture output + exit code. @@ -267,7 +267,7 @@ def to_cmd_str(cmd): dry_run_msg(msg, silent=silent) return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) start_time = datetime.now() if not hidden: @@ -344,7 +344,7 @@ def to_cmd_str(cmd): raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) # always log command output cmd_name = cmd_str.split(' ')[0] diff --git a/test/framework/run.py b/test/framework/run.py index 9a9b02d767..c86d8635ed 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -911,7 +911,7 @@ def test_run_shell_cmd_cache(self): with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None, - thread_id=None) + thread_id=None, task_id=None) run_shell_cmd.update_cache({(cmd, None): cached_res}) res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) @@ -931,7 +931,7 @@ def test_run_shell_cmd_cache(self): with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None, - thread_id=None) + thread_id=None, task_id=None) run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 5f4b62f09c..6d8395a5fc 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -341,7 +341,7 @@ def mocked_run_shell_cmd(cmd, **kwargs): } if cmd in known_cmds: return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(), - out_file=None, err_file=None, thread_id=None) + out_file=None, err_file=None, thread_id=None, task_id=None) else: return run_shell_cmd(cmd, **kwargs) @@ -774,7 +774,7 @@ def test_gcc_version_darwin(self): out = "Apple LLVM version 7.0.0 (clang-700.1.76)" cwd = os.getcwd() mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd, - out_file=None, err_file=None, thread_id=None) + out_file=None, err_file=None, thread_id=None, task_id=None) st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) From 77e077e372c300d698f2f7da7cf005ea7b555977 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 20:00:19 +0100 Subject: [PATCH 067/430] fix error reporting when test step fails --- easybuild/framework/easyblock.py | 5 ++++- test/framework/toy_build.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f6077dd5ce..a684b74347 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2736,7 +2736,10 @@ def _test_step(self): try: self.test_step() except RunShellCmdError as err: - self.report_test_failure(err) + err.print() + ec_path = os.path.basename(self.cfg.path) + error_msg = f"shell command '{err.cmd_name} ...' failed in test step for {ec_path}" + self.report_test_failure(error_msg) def stage_install_step(self): """ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6ba47d3587..82bc052a93 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -4124,6 +4124,22 @@ def test_toy_build_info_msg(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_toy_failing_test_step(self): + """ + Test behaviour when test step fails, using toy easyconfig. + """ + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nruntest = "false"' + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + error_pattern = r"shell command 'false \.\.\.' failed in test step" + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_test_toy_build_with_output, + ec_file=test_ec, raise_error=True) + def test_eb_crash(self): """ Test behaviour when EasyBuild crashes, for example due to a buggy hook From ea562b8ead14500f1112bf4b4b8cbbbf20d3dcd2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 22:45:17 +0100 Subject: [PATCH 068/430] fix broken tests --- test/framework/filetools.py | 46 ++++++++++++++++++------------------- test/framework/run.py | 2 +- test/framework/toy_build.py | 4 ++-- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 800ee9ba0b..8b72aefc4c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2290,7 +2290,7 @@ def test_extract_file(self): self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0')) - self.assertTrue(re.search('running command "tar xzf .*/toy-0.0.tar.gz"', txt)) + self.assertTrue(re.search('running shell command "tar xzf .*/toy-0.0.tar.gz"', txt)) with self.mocked_stdout_stderr(): path = ft.extract_file(toy_tarball, self.test_prefix, forced=True, change_into_dir=False) @@ -2314,7 +2314,7 @@ def test_extract_file(self): self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertTrue(os.path.samefile(os.getcwd(), self.test_prefix)) self.assertFalse(stderr) - self.assertTrue("running command" in stdout) + self.assertTrue("running shell command" in stdout) # check whether disabling trace output works with self.mocked_stdout_stderr(): @@ -2800,18 +2800,18 @@ def run_check(): } git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() git_config['clone_into'] = 'test123' expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', r" \(in /.*\)", ]) % git_repo run_check() @@ -2819,19 +2819,19 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - ' running command "git clone --depth 1 --branch tag_for_tests --recursive' + ' running shell command "git clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) % git_repo run_check() @@ -2841,11 +2841,11 @@ def run_check(): 'submodule."sha1".active=false', ] expected = '\n'.join([ - ' running command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false' + ' running shell command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false' + ' clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) % git_repo run_check() @@ -2854,9 +2854,9 @@ def run_check(): git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in /.*\)", ]) % git_repo run_check() @@ -2865,23 +2865,23 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in /.*\)", - r' running command "git checkout 8456f86 && git submodule update --init --recursive"', + r' running shell command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in /.*/testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - ' running command "git checkout 8456f86 && git submodule update --init --recursive' + ' running shell command "git checkout 8456f86 && git submodule update --init --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\'"', r" \(in /.*/testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) % git_repo run_check() @@ -2889,11 +2889,11 @@ def run_check(): del git_config['recursive'] del git_config['recurse_submodules'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in /.*\)", - r' running command "git checkout 8456f86"', + r' running shell command "git checkout 8456f86"', r" \(in /.*/testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() diff --git a/test/framework/run.py b/test/framework/run.py index c86d8635ed..1c9b0b2562 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1273,7 +1273,7 @@ def test_run_shell_cmd_async(self): os.environ['TEST'] = 'test123' env = os.environ.copy() - test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" + test_cmd = "echo 'sleeping...'; sleep 3; echo $TEST" task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) # change value of $TEST to check that command is completed with correct environment diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 936fd09c4c..6ab83f67fc 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2975,11 +2975,11 @@ def test_toy_build_trace(self): r"^== fetching files\.\.\.\n >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$", r"^ >> applying patch toy-0\.0_fix-silly-typo-in-printf-statement\.patch$", r'\n'.join([ - r"^ >> running command:", + r"^ >> running shell command:", + r"\tgcc toy.c -o toy\n" r"\t\[started at: .*\]", r"\t\[working dir: .*\]", r"\t\[output saved to .*\]", - r"\tgcc toy.c -o toy\n" r'', ]), r" >> command completed: exit 0, ran in .*", From a19776a5e5896fc3c98f594a6aca410ed941f652 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 23:04:01 +0100 Subject: [PATCH 069/430] use threading.get_ident as fallback for threading.get_native_id for Python < 3.8 --- easybuild/tools/run.py | 10 ++++++++-- test/framework/run.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 852d13a966..986c541538 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -45,11 +45,17 @@ import subprocess import sys import tempfile -import threading import time from collections import namedtuple from datetime import datetime +try: + # get_native_id is only available in Python >= 3.8 + from threading import get_native_id as get_thread_id +except ImportError: + # get_ident is available in Python >= 3.3 + from threading import get_ident as get_thread_id + import easybuild.tools.asyncprocess as asyncprocess from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since @@ -235,7 +241,7 @@ def to_cmd_str(cmd): thread_id = None if asynchronous: - thread_id = threading.get_native_id() + thread_id = get_thread_id() _log.info(f"Initiating running of shell command '{cmd_str}' via thread with ID {thread_id}") # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely diff --git a/test/framework/run.py b/test/framework/run.py index 1c9b0b2562..c86d8635ed 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1273,7 +1273,7 @@ def test_run_shell_cmd_async(self): os.environ['TEST'] = 'test123' env = os.environ.copy() - test_cmd = "echo 'sleeping...'; sleep 3; echo $TEST" + test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) # change value of $TEST to check that command is completed with correct environment From 75fa3ee8e209beee3e291ad77ddfe0b5101e8475 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:42:28 +0000 Subject: [PATCH 070/430] do not test deprecated module tools --- .github/workflows/unit_tests.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 225b3f85c6..324b64630b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -14,8 +14,6 @@ jobs: runs-on: ubuntu-20.04 outputs: lmod8: Lmod-8.7.6 - modulesTcl: modules-tcl-1.147 - modules3: modules-3.2.10 modules4: modules-4.1.4 steps: - run: "true" @@ -29,8 +27,6 @@ jobs: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context - ${{needs.setup.outputs.lmod8}} - - ${{needs.setup.outputs.modulesTcl}} - - ${{needs.setup.outputs.modules3}} - ${{needs.setup.outputs.modules4}} lc_all: [""] include: @@ -156,11 +152,7 @@ jobs: export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH eb --version # tell EasyBuild which modules tool is available - if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl - elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesC - elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then + if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then export EASYBUILD_MODULES_TOOL=EnvironmentModules else export EASYBUILD_MODULES_TOOL=Lmod From fffba9694a1440e560ed36c3201f118405c59a3e Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:05:40 +0000 Subject: [PATCH 071/430] test with non-deprecated module tool --- test/framework/modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index f8226e8d33..b28564885c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1531,10 +1531,10 @@ def test_modulecmd_strip_source(self): '#!/bin/bash', # if last argument (${!#})) is --version, print version 'if [ x"${!#}" == "x--version" ]; then', - ' echo 3.2.10', + ' echo 4.2.10', # otherwise, echo Python commands: set $TEST123 and include a faulty 'source' command 'else', - ' echo "source /opt/cray/pe/modules/3.2.10.6/init/bash"', + ' echo "source /opt/cray/pe/modules/4.2.10.6/init/bash"', " echo \"os.environ['TEST123'] = 'test123'\"", 'fi', ]) @@ -1543,7 +1543,7 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) - modtool = EnvironmentModulesC() + modtool = EnvironmentModules() modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') From 7fffcba6a279a8ea3803f162154810a334236d1f Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:08:45 +0000 Subject: [PATCH 072/430] version with a . --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 09524d5ee1..aa8cbf9381 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1243,7 +1243,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC): COMMAND_SHELL = ['tclsh'] VERSION_OPTION = '' REQ_VERSION = None - DEPR_VERSION = '9999' + DEPR_VERSION = '9999.9' VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' def set_path_env_var(self, key, paths): From fd18449daae861c5c8b522942d2346dd98fb674f Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:00:18 +0000 Subject: [PATCH 073/430] set a DEPR_VERSION for EnvironmentModules --- easybuild/tools/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index aa8cbf9381..75a1022b50 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1318,6 +1318,7 @@ class EnvironmentModules(EnvironmentModulesTcl): COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' + DEPR_VERSION = '4.0.0' # needs to be set as EnvironmentModules inherits from EnvironmentModulesTcl MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' From 8d8029d02fe241e495d2a96311e2c169fb8f0a9d Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:12:57 +0000 Subject: [PATCH 074/430] uses old module tool, so allow deprecated --- test/framework/modules.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index b28564885c..f1a242f54a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1531,10 +1531,10 @@ def test_modulecmd_strip_source(self): '#!/bin/bash', # if last argument (${!#})) is --version, print version 'if [ x"${!#}" == "x--version" ]; then', - ' echo 4.2.10', + ' echo 3.2.10', # otherwise, echo Python commands: set $TEST123 and include a faulty 'source' command 'else', - ' echo "source /opt/cray/pe/modules/4.2.10.6/init/bash"', + ' echo "source /opt/cray/pe/modules/3.2.10.6/init/bash"', " echo \"os.environ['TEST123'] = 'test123'\"", 'fi', ]) @@ -1543,7 +1543,8 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) - modtool = EnvironmentModules() + self.allow_deprecated_behaviour() + modtool = EnvironmentModulesC() modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') From cc2a47817e41c6fd62b03ec40976c1a6f6f9897b Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:54:57 +0000 Subject: [PATCH 075/430] and capture the dep. warning --- test/framework/modules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index f1a242f54a..a849148bdf 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1544,8 +1544,9 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) self.allow_deprecated_behaviour() - modtool = EnvironmentModulesC() - modtool.run_module('load', 'test123') + with self.mocked_stdout_stderr(): + modtool = EnvironmentModulesC() + modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') def test_get_setenv_value_from_modulefile(self): From 7315b9f7a6e47515bb255bf2aa0da8cb22595ff2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 24 Jan 2024 14:25:47 +0100 Subject: [PATCH 076/430] improve findPythonDeps to recognize non-canonical package names For e.g. `ruamel.yaml` the canonical name is `ruamel-yaml` but the package name as recorded is still `ruamel.yaml`. So if the search using the canonical name didn't find anything try again with the original name before failing. Tested on `maggma==0.60.2` --- easybuild/scripts/findPythonDeps.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index d6e496a048..c3607e47c7 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -96,10 +96,13 @@ def get_dep_tree(package_spec, verbose): def find_deps(pkgs, dep_tree): """Recursively resolve dependencies of the given package(s) and return them""" res = [] - for pkg in pkgs: - pkg = canonicalize_name(pkg) + for orig_pkg in pkgs: + pkg = canonicalize_name(orig_pkg) matching_entries = [entry for entry in dep_tree if pkg in (entry['package']['package_name'], entry['package']['key'])] + if not matching_entries: + matching_entries = [entry for entry in dep_tree + if orig_pkg in (entry['package']['package_name'], entry['package']['key'])] if not matching_entries: raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree)) if len(matching_entries) > 1: From e91fa2978f485c1a887f674b696f5437beec6124 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Jan 2024 09:52:31 +0100 Subject: [PATCH 077/430] continue to execute deprecated run() methods still found on custom easyblock --- easybuild/framework/easyblock.py | 11 ++++++++++- easybuild/framework/extension.py | 6 +----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3a42b40a0f..9c543ad8f1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1941,7 +1941,16 @@ def install_extensions_sequential(self, install=True): try: ext.pre_install_extension() with self.module_generator.start_module_creation(): - txt = ext.install_extension() + parent_ext_obj = super(ext.__class__, ext) + if ext.run.__hash__() != parent_ext_obj.run.__hash__(): + # DEPRECATED: easyblock has custom run() method + self.log.deprecated( + "Extension.run() is deprecated, use Extension.install_extension() instead.", + '6.0', + ) + txt = ext.run() + else: + txt = ext.install_extension() if txt: self.module_extra_extensions += txt ext.post_install_extension() diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 82f6063a52..171c6265ff 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -189,12 +189,8 @@ def pre_install_extension(self): def run(self, *args, **kwargs): """ - [DEPRECATED] Actual installation of an extension. + [DEPRECATED][6.0] Actual installation of an extension. """ - self.log.deprecated( - "Extension.run() is deprecated, use Extension.install_extension() instead.", - '6.0', - ) self.install_extension(*args, **kwargs) def install_extension(self, *args, **kwargs): From cf5c10b77bf34b5ce791b62f997a70aabba6828d Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Jan 2024 13:00:05 +0100 Subject: [PATCH 078/430] add Extension.install_extension_substep wrapping all substeps to detect and use any custom deprecated methods in the Easyblock --- easybuild/framework/easyblock.py | 21 ++++--------- easybuild/framework/extension.py | 51 +++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9c543ad8f1..855c386e42 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1939,21 +1939,12 @@ def install_extensions_sequential(self, install=True): # actual installation of the extension if install: try: - ext.pre_install_extension() + ext.install_extension_substep("pre_install_extension") with self.module_generator.start_module_creation(): - parent_ext_obj = super(ext.__class__, ext) - if ext.run.__hash__() != parent_ext_obj.run.__hash__(): - # DEPRECATED: easyblock has custom run() method - self.log.deprecated( - "Extension.run() is deprecated, use Extension.install_extension() instead.", - '6.0', - ) - txt = ext.run() - else: - txt = ext.install_extension() + txt = ext.install_extension_substep("install_extension") if txt: self.module_extra_extensions += txt - ext.post_install_extension() + ext.install_extension_substep("post_install_extension") finally: if not self.dry_run: ext_duration = datetime.now() - start_time @@ -2014,7 +2005,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): for ext in running_exts[:]: if self.dry_run or ext.async_cmd_check(): self.log.info("Installation of %s completed!", ext.name) - ext.post_install_extension() + ext.install_extension_substep("post_install_extension") running_exts.remove(ext) installed_ext_names.append(ext.name) update_exts_progress_bar_helper(running_exts, 1) @@ -2085,8 +2076,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size): ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, rpath_filter_dirs=self.rpath_filter_dirs) if install: - ext.pre_install_extension() - ext.install_extension_async() + ext.install_extension_substep("pre_install_extension") + ext.install_extension_substep("install_extension_async") running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar_helper(running_exts, 0) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 171c6265ff..3073d18165 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -173,12 +173,9 @@ def version(self): def prerun(self): """ - [DEPRECATED] Stuff to do before installing a extension. + [DEPRECATED][6.0] Stuff to do before installing a extension. """ - self.log.deprecated( - "Extension.prerun() is deprecated, use Extension.pre_install_extension() instead.", - '6.0', - ) + # Deprecation warning triggered by Extension.install_extension_substep() self.pre_install_extension() def pre_install_extension(self): @@ -191,6 +188,7 @@ def run(self, *args, **kwargs): """ [DEPRECATED][6.0] Actual installation of an extension. """ + # Deprecation warning triggered by Extension.install_extension_substep() self.install_extension(*args, **kwargs) def install_extension(self, *args, **kwargs): @@ -217,12 +215,9 @@ def install_extension_async(self, *args, **kwargs): def postrun(self): """ - [DEPRECATED] Stuff to do after installing a extension. + [DEPRECATED][6.0] Stuff to do after installing a extension. """ - self.log.deprecated( - "Extension.postrun() is deprecated, use Extension.post_install_extension() instead.", - '6.0', - ) + # Deprecation warning triggered by Extension.install_extension_substep() self.post_install_extension() def post_install_extension(self): @@ -231,6 +226,42 @@ def post_install_extension(self): """ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) + def install_extension_substep(self, substep): + """ + Carry out extension installation substep allowing use of deprecated + methods on those extensions using an older EasyBlock + """ + deprecated = { + 'pre_install_extension': 'prerun', + 'install_extension': 'run', + 'install_extension_async': 'run_async', + 'post_install_extension': 'postrun', + } + + if substep not in deprecated: + raise EasyBuildError("Unknown extension installation substep: %s", substep) + + try: + ext_substep = getattr(self, deprecated[substep]) + parent_obj = super(self.__class__, self) + parent_substep = getattr(parent_obj, deprecated[substep]) + except AttributeError: + self.log.debug("Easyblock does not provide deprecated method for installation substep: %s", substep) + ext_substep = getattr(self, substep) + else: + if ext_substep.__hash__() == parent_substep.__hash__(): + # Deprecated method is present in parent, but no custom method in child Easyblock + ext_substep = getattr(self, substep) + else: + # Custom deprecated method used by child Easyblock + self.log.debug("Easyblock provides custom deprecated method for installation substep: %s", substep) + self.log.deprecated( + f"Extension.{deprecated[substep]}() is deprecated, use Extension.{substep}() instead.", + "6.0", + ) + + ext_substep() + def async_cmd_start(self, cmd, inp=None): """ Start installation asynchronously using specified command. From 7cd6732488517af4f9d535a811803dcbeeabccfe Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Jan 2024 18:49:31 +0100 Subject: [PATCH 079/430] Extension.install_extension_substep returns outcome of install extension substep --- easybuild/framework/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 3073d18165..53020d6cf0 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -260,7 +260,7 @@ def install_extension_substep(self, substep): "6.0", ) - ext_substep() + return ext_substep() def async_cmd_start(self, cmd, inp=None): """ From 6074b5511befe2917aa53cece9ef037acfea3d43 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Jan 2024 18:56:34 +0100 Subject: [PATCH 080/430] remove duplicate deprecation warning on Extension.run_async() --- easybuild/framework/extension.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 53020d6cf0..f807c1f6b1 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -199,12 +199,9 @@ def install_extension(self, *args, **kwargs): def run_async(self, *args, **kwargs): """ - [DEPRECATED] Asynchronous installation of an extension. + [DEPRECATED][6.0] Asynchronous installation of an extension. """ - self.log.deprecated( - "Extension.run_async() is deprecated, use Extension.install_extension_async() instead.", - '6.0', - ) + # Deprecation warning triggered by Extension.install_extension_substep() self.install_extension_async(*args, **kwargs) def install_extension_async(self, *args, **kwargs): From 36febd8739ab37900577db8d0cdd13ea0e1f48fa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 29 Jan 2024 15:18:56 +0100 Subject: [PATCH 081/430] use run_shell_cmd to install extensions in parallel --- easybuild/framework/easyblock.py | 33 ++++++++----- easybuild/framework/extension.py | 48 +------------------ easybuild/tools/run.py | 1 + .../easyblocks/generic/toy_extension.py | 11 +++-- .../sandbox/easybuild/easyblocks/t/toy.py | 6 ++- 5 files changed, 36 insertions(+), 63 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4f45dbbaab..e2d84c162e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -87,7 +87,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import RunShellCmdError, run_shell_cmd +from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -1961,6 +1961,8 @@ def install_extensions_parallel(self, install=True): """ self.log.info("Installing extensions in parallel...") + thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) + running_exts = [] installed_ext_names = [] @@ -1997,16 +1999,23 @@ def update_exts_progress_bar_helper(running_exts, progress_size): # check for extension installations that have completed if running_exts: - self.log.info("Checking for completed extension installations (%d running)...", len(running_exts)) + self.log.info(f"Checking for completed extension installations ({len(running_exts)} running)...") for ext in running_exts[:]: - if self.dry_run or ext.async_cmd_check(): - self.log.info("Installation of %s completed!", ext.name) - ext.postrun() - running_exts.remove(ext) - installed_ext_names.append(ext.name) - update_exts_progress_bar_helper(running_exts, 1) + if self.dry_run or ext.async_cmd_task.done(): + res = ext.async_cmd_task.result() + if res.exit_code == 0: + self.log.info(f"Installation of extension {ext.name} completed!") + # run post-install method for extension from same working dir as installation of extension + cwd = change_dir(res.work_dir) + ext.postrun() + change_dir(cwd) + running_exts.remove(ext) + installed_ext_names.append(ext.name) + update_exts_progress_bar_helper(running_exts, 1) + else: + raise_run_shell_cmd_error(res) else: - self.log.debug("Installation of %s is still running...", ext.name) + self.log.debug(f"Installation of extension {ext.name} is still running...") # try to start as many extension installations as we can, taking into account number of available cores, # but only consider first 100 extensions still in the queue @@ -2073,9 +2082,9 @@ def update_exts_progress_bar_helper(running_exts, progress_size): rpath_filter_dirs=self.rpath_filter_dirs) if install: ext.prerun() - ext.run_async() + ext.async_cmd_task = ext.run_async(thread_pool) running_exts.append(ext) - self.log.info("Started installation of extension %s in the background...", ext.name) + self.log.info(f"Started installation of extension {ext.name} in the background...") update_exts_progress_bar_helper(running_exts, 0) # print progress info after every iteration (unless that info is already shown via progress bar) @@ -2088,6 +2097,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size): running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) + thread_pool.shutdown() + # # MISCELLANEOUS UTILITY FUNCTIONS # diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index d30242495c..9f099eb74c 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -42,7 +42,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import check_async_cmd, run_cmd, run_shell_cmd +from easybuild.tools.run import run_shell_cmd def resolve_exts_filter_template(exts_filter, ext): @@ -150,12 +150,7 @@ def __init__(self, mself, ext, extra_params=None): self.sanity_check_module_loaded = False self.fake_mod_data = None - self.async_cmd_info = None - self.async_cmd_output = None - self.async_cmd_check_cnt = None - # initial read size should be relatively small, - # to avoid hanging for a long time until desired output is available in async_cmd_check - self.async_cmd_read_size = 1024 + self.async_cmd_task = None @property def name(self): @@ -195,44 +190,6 @@ def postrun(self): """ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) - def async_cmd_start(self, cmd, inp=None): - """ - Start installation asynchronously using specified command. - """ - self.async_cmd_output = '' - self.async_cmd_check_cnt = 0 - self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True) - - def async_cmd_check(self): - """ - Check progress of installation command that was started asynchronously. - - :return: True if command completed, False otherwise - """ - if self.async_cmd_info is None: - raise EasyBuildError("No installation command running asynchronously for %s", self.name) - elif self.async_cmd_info is False: - self.log.info("No asynchronous command was started for extension %s", self.name) - return True - else: - self.log.debug("Checking on installation of extension %s...", self.name) - # use small read size, to avoid waiting for a long time until sufficient output is produced - res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size) - self.async_cmd_output += res['output'] - if res['done']: - self.log.info("Installation of extension %s completed!", self.name) - self.async_cmd_info = None - else: - self.async_cmd_check_cnt += 1 - self.log.debug("Installation of extension %s still running (checked %d times)", - self.name, self.async_cmd_check_cnt) - # increase read size after sufficient checks, - # to avoid that installation hangs due to output buffer filling up... - if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): - self.async_cmd_read_size *= 2 - - return res['done'] - @property def required_deps(self): """Return list of required dependencies for this extension.""" @@ -273,7 +230,6 @@ def sanity_check_step(self): self.log.info("modulename set to False for '%s' extension, so skipping sanity check", self.name) elif exts_filter: cmd, stdin = resolve_exts_filter_template(exts_filter, self) - # set log_ok to False so we can catch the error instead of run_cmd cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) if cmd_res.exit_code: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 986c541538..95391f3821 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -207,6 +207,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param output_file: collect command output in temporary output file :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None) :param asynchronous: indicate that command is being run asynchronously + :param task_id: task ID for specified shell command (included in return value) :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index de8c4c89c1..15fe5773aa 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -27,6 +27,7 @@ @author: Kenneth Hoste (Ghent University) """ +import os from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock @@ -81,22 +82,24 @@ def prerun(self): super(Toy_Extension, self).run(unpack_src=True) EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) - def run_async(self): + def run_async(self, thread_pool): """ Install toy extension asynchronously. """ + task_id = f'ext_{self.name}_{self.version}' if self.src: cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) - self.async_cmd_start(cmd) else: - self.async_cmd_info = False + cmd = f"echo 'no sources for {self.name}'" + + return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), + fail_on_error=False, task_id=task_id) def postrun(self): """ Wrap up installation of toy extension. """ super(Toy_Extension, self).postrun() - EB_toy.install_step(self.master, name=self.name) def sanity_check_step(self, *args, **kwargs): diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index a1454435e3..fb842bfe06 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -163,12 +163,14 @@ def run(self): """ self.build_step() - def run_async(self): + def run_async(self, thread_pool): """ Asynchronous installation of toy as extension. """ cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) - self.async_cmd_start(cmd) + task_id = f'ext_{self.name}_{self.version}' + return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), + fail_on_error=False, task_id=task_id) def postrun(self): """ From cedd5ab74a422b2cbaa026d47204a55ee4be540b Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 3 Feb 2024 19:47:46 +0000 Subject: [PATCH 082/430] enable rpath by default --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index be5cf94f75..cb6ec2022f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -489,7 +489,7 @@ def override_options(self): 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " "which must be linked in all installed binaries/libraries", 'strlist', 'extend', None), - 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', False), + 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', True), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", None, 'store', None), From 54b36337a187bf1f6605216d82d2ee455fee5dbc Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:44:39 +0000 Subject: [PATCH 083/430] disable rpath in two tests --- test/framework/options.py | 1 + test/framework/toolchain.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index ca8a582365..5d87695661 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4121,6 +4121,7 @@ def test_extended_dry_run(self): '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--debug', + '--disable-rpath', ] msg_regexs = [ diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 1c8c1771a6..918b344dcf 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2269,6 +2269,7 @@ def test_compiler_cache(self): "--force", "--debug", "--disable-cleanup-tmpdir", + "--disable-rpath", ] ccache = which('ccache') From 0bee2eca2b252d520ab1f24cfe18400718bc9cec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Feb 2024 13:50:38 +0100 Subject: [PATCH 084/430] fix EasyBlock in deprecation log message produced by EasyBlock.install_extensions --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 855c386e42..e6b39c009a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1867,7 +1867,7 @@ def skip_extensions_parallel(self, exts_filter): def install_extensions(self, *args, **kwargs): """[DEPRECATED] Install extensions.""" self.log.deprecated( - "Easyblock.install_extensions() is deprecated, use Easyblock.install_all_extensions() instead.", + "EasyBlock.install_extensions() is deprecated, use EasyBlock.install_all_extensions() instead.", '6.0', ) self.install_all_extensions(*args, **kwargs) @@ -4479,7 +4479,7 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): for easyblock_class in inspect.getmro(type(easyblock_instance)): easyblock_path = inspect.getsourcefile(easyblock_class) # if we reach EasyBlock, Extension or ExtensionEasyBlock class, we are done - # (Extension and ExtensionEasyblock are hardcoded to avoid a cyclical import) + # (Extension and ExtensionEasyBlock are hardcoded to avoid a cyclical import) if easyblock_class.__name__ in [EasyBlock.__name__, 'Extension', 'ExtensionEasyBlock']: break else: From e3845799fef67a0c7215a06905ef5867626dc34c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Feb 2024 13:52:03 +0100 Subject: [PATCH 085/430] enhance install_extension_substep to support passing down (named) arguments --- easybuild/framework/extension.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index f807c1f6b1..a68f22a38d 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -223,27 +223,30 @@ def post_install_extension(self): """ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) - def install_extension_substep(self, substep): + def install_extension_substep(self, substep, *args, **kwargs): """ Carry out extension installation substep allowing use of deprecated methods on those extensions using an older EasyBlock """ - deprecated = { + substeps_mapping = { 'pre_install_extension': 'prerun', 'install_extension': 'run', 'install_extension_async': 'run_async', 'post_install_extension': 'postrun', } - if substep not in deprecated: + deprecated_method = substeps_mapping.get(substep) + if deprecated_method is None: raise EasyBuildError("Unknown extension installation substep: %s", substep) try: - ext_substep = getattr(self, deprecated[substep]) + ext_substep = getattr(self, deprecated_method) parent_obj = super(self.__class__, self) - parent_substep = getattr(parent_obj, deprecated[substep]) + parent_substep = getattr(parent_obj, deprecated_method) except AttributeError: - self.log.debug("Easyblock does not provide deprecated method for installation substep: %s", substep) + log_msg = f"EasyBlock does not implement deprecated method '{deprecated_method}' " + log_msg += f"for installation substep {substep}" + self.log.debug(log_msg) ext_substep = getattr(self, substep) else: if ext_substep.__hash__() == parent_substep.__hash__(): @@ -251,13 +254,13 @@ def install_extension_substep(self, substep): ext_substep = getattr(self, substep) else: # Custom deprecated method used by child Easyblock - self.log.debug("Easyblock provides custom deprecated method for installation substep: %s", substep) + self.log.debug(f"EasyBlock provides custom deprecated method for installation substep: {substep}") self.log.deprecated( f"Extension.{deprecated[substep]}() is deprecated, use Extension.{substep}() instead.", "6.0", ) - return ext_substep() + return ext_substep(*args, **kwargs) def async_cmd_start(self, cmd, inp=None): """ From 166227862602b04dfdb75cd64438f30785030056 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 7 Feb 2024 16:56:45 +0100 Subject: [PATCH 086/430] fix variable rename in install_extension_substep error message --- easybuild/framework/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index a68f22a38d..b4570f0f75 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -256,7 +256,7 @@ def install_extension_substep(self, substep, *args, **kwargs): # Custom deprecated method used by child Easyblock self.log.debug(f"EasyBlock provides custom deprecated method for installation substep: {substep}") self.log.deprecated( - f"Extension.{deprecated[substep]}() is deprecated, use Extension.{substep}() instead.", + f"Extension.{deprecated_method}() is deprecated, use Extension.{substep}() instead.", "6.0", ) From 92e9fe1ea235dec4b5859c9d6b27354bc763cc9e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Feb 2024 18:11:35 +0100 Subject: [PATCH 087/430] fix broken tests after symlinking lib -> lib64 before postinstallcmds --- test/framework/toy_build.py | 42 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f44182e2b8..f4ee8395c1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2414,7 +2414,14 @@ def test_sanity_check_paths_lib64(self): # modify test easyconfig: move lib/libtoy.a to lib64/libtoy.a ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", ectxt) - postinstallcmd = "mkdir %(installdir)s/lib64 && mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a" + postinstallcmd = ' && '.join([ + # remove lib64 symlink (if it's there) + "rm -f %(installdir)s/lib64", + # create empty lib64 dir + "mkdir %(installdir)s/lib64", + # move libtoy.a + "mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a", + ]) ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt) test_ec = os.path.join(self.test_prefix, 'toy-0.0.eb') @@ -3829,7 +3836,6 @@ def test_toy_build_lib_lib64_symlink(self): toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') test_ec_txt = read_file(toy_ec) - test_ec_txt += "\npostinstallcmds += ['mv %(installdir)s/lib %(installdir)s/lib64']" test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -3842,30 +3848,30 @@ def test_toy_build_lib_lib64_symlink(self): lib_path = os.path.join(toy_installdir, 'lib') lib64_path = os.path.join(toy_installdir, 'lib64') - # lib64 subdir exists, is not a symlink - self.assertExists(lib64_path) - self.assertTrue(os.path.isdir(lib64_path)) - self.assertFalse(os.path.islink(lib64_path)) - - # lib subdir is a symlink to lib64 subdir + # lib subdir exists, is not a symlink self.assertExists(lib_path) self.assertTrue(os.path.isdir(lib_path)) - self.assertTrue(os.path.islink(lib_path)) - self.assertTrue(os.path.samefile(lib_path, lib64_path)) + self.assertFalse(os.path.islink(lib_path)) - # lib symlink should point to a relative path - self.assertFalse(os.path.isabs(os.readlink(lib_path))) + # lib64 subdir is a symlink to lib subdir + self.assertExists(lib64_path) + self.assertTrue(os.path.isdir(lib64_path)) + self.assertTrue(os.path.islink(lib64_path)) + self.assertTrue(os.path.samefile(lib64_path, lib_path)) + + # lib64 symlink should point to a relative path + self.assertFalse(os.path.isabs(os.readlink(lib64_path))) # cleanup and try again with --disable-lib-lib64-symlink remove_dir(self.test_installpath) with self.mocked_stdout_stderr(): - self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib-lib64-symlink']) + self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink']) - self.assertExists(lib64_path) - self.assertNotExists(lib_path) - self.assertNotIn('lib', os.listdir(toy_installdir)) - self.assertTrue(os.path.isdir(lib64_path)) - self.assertFalse(os.path.islink(lib64_path)) + self.assertExists(lib_path) + self.assertNotExists(lib64_path) + self.assertNotIn('lib64', os.listdir(toy_installdir)) + self.assertTrue(os.path.isdir(lib_path)) + self.assertFalse(os.path.islink(lib_path)) def test_toy_build_sanity_check_linked_libs(self): """Test sanity checks for banned/requires libraries.""" From 612b1c0448db922adaed55c1f14ebb45f8bff2a8 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 7 Feb 2024 18:29:51 +0000 Subject: [PATCH 088/430] rpath off on mac --- easybuild/tools/options.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index cb6ec2022f..197514eb07 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -103,8 +103,8 @@ from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories -from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family -from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_system_info +from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family +from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_os_type, get_system_info from easybuild.tools.version import this_is_easybuild @@ -131,6 +131,8 @@ def terminal_supports_colors(stream): DEFAULT_LIST_PR_ORDER = GITHUB_PR_ORDER_CREATED DEFAULT_LIST_PR_DIREC = GITHUB_PR_DIRECTION_DESC +RPATH_DEFAULT = False if get_os_type() == DARWIN else True + _log = fancylogger.getLogger('options', fname=False) @@ -489,7 +491,7 @@ def override_options(self): 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " "which must be linked in all installed binaries/libraries", 'strlist', 'extend', None), - 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', True), + 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', RPATH_DEFAULT), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", None, 'store', None), From 44b2b9f6965a9eba1182b57de948847284590b62 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 8 Feb 2024 03:12:14 +0100 Subject: [PATCH 089/430] compare deprecated method qualified name instead of hashes in Extension.install_extension_substep --- easybuild/framework/extension.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index b4570f0f75..a0b9b59ea0 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -235,32 +235,35 @@ def install_extension_substep(self, substep, *args, **kwargs): 'post_install_extension': 'postrun', } - deprecated_method = substeps_mapping.get(substep) - if deprecated_method is None: + deprecated_substep = substeps_mapping.get(substep) + if deprecated_substep is None: raise EasyBuildError("Unknown extension installation substep: %s", substep) try: - ext_substep = getattr(self, deprecated_method) - parent_obj = super(self.__class__, self) - parent_substep = getattr(parent_obj, deprecated_method) + substep_method = getattr(self, deprecated_substep) except AttributeError: - log_msg = f"EasyBlock does not implement deprecated method '{deprecated_method}' " + log_msg = f"EasyBlock does not implement deprecated method '{deprecated_substep}' " log_msg += f"for installation substep {substep}" self.log.debug(log_msg) - ext_substep = getattr(self, substep) + substep_method = getattr(self, substep) else: - if ext_substep.__hash__() == parent_substep.__hash__(): - # Deprecated method is present in parent, but no custom method in child Easyblock - ext_substep = getattr(self, substep) + # Qualified method name contains class defining the method (PEP 3155) + substep_method_name = substep_method.__qualname__ + self.log.debug(f"Found deprecated method in EasyBlock: {substep_method_name}") + + base_method_name = f"Extension.{substep}" + if substep_method_name == base_method_name: + # No custom method in child Easyblock, deprecated method is defined by base Extension class + # Switch to non-deprecated substep method + substep_method = getattr(self, substep) else: # Custom deprecated method used by child Easyblock - self.log.debug(f"EasyBlock provides custom deprecated method for installation substep: {substep}") self.log.deprecated( - f"Extension.{deprecated_method}() is deprecated, use Extension.{substep}() instead.", + f"{substep_method_name}() is deprecated, use {substep}() instead.", "6.0", ) - return ext_substep(*args, **kwargs) + return substep_method(*args, **kwargs) def async_cmd_start(self, cmd, inp=None): """ From 4ded825633071051087c0e5d66b6108b7a219382 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 8 Feb 2024 09:16:13 +0100 Subject: [PATCH 090/430] fix check on deprecated methods on base Extension class --- easybuild/framework/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index a0b9b59ea0..19f8a77730 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -251,7 +251,7 @@ def install_extension_substep(self, substep, *args, **kwargs): substep_method_name = substep_method.__qualname__ self.log.debug(f"Found deprecated method in EasyBlock: {substep_method_name}") - base_method_name = f"Extension.{substep}" + base_method_name = f"Extension.{deprecated_substep}" if substep_method_name == base_method_name: # No custom method in child Easyblock, deprecated method is defined by base Extension class # Switch to non-deprecated substep method From 0960bc4f459826e6f378ad963ebe79d1b9474e8e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 8 Feb 2024 13:14:50 +0100 Subject: [PATCH 091/430] clean up log file of EasyBlock instance in check_sha256_checksums --- easybuild/framework/easyconfig/tools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index c6649c383b..73393040c6 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -701,7 +701,9 @@ def check_sha256_checksums(ecs, whitelist=None): continue eb_class = get_easyblock_class(ec['easyblock'], name=ec['name']) - checksum_issues.extend(eb_class(ec).check_checksums()) + eb = eb_class(ec) + checksum_issues.extend(eb.check_checksums()) + eb.close_log() return checksum_issues From 22e12b4c068028a34ed2a85424b330bce96a090d Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:43:45 +0000 Subject: [PATCH 092/430] allow only alphanumeric characters in the output filename --- easybuild/tools/run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 92c92a1b41..e525f8f95a 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -42,6 +42,7 @@ import re import signal import shutil +import string import subprocess import sys import tempfile @@ -233,7 +234,6 @@ def to_cmd_str(cmd): work_dir = os.getcwd() cmd_str = to_cmd_str(cmd) - cmd_name = os.path.basename(cmd_str.split(' ')[0]) # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): @@ -244,6 +244,9 @@ def to_cmd_str(cmd): if output_file: toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') os.makedirs(toptmpdir, exist_ok=True) + # restrict the allowed characters in the name of the output_file + allowed_chars = string.ascii_letters + string.digits + cmd_name = ''.join([c for c in os.path.basename(cmd_str.split(' ')[0]) if c in allowed_chars]) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') From faf8f1439b51f388fad600021a316df1acd9ab32 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Thu, 15 Feb 2024 11:14:33 +0000 Subject: [PATCH 093/430] functionalise the simplification and add tests --- easybuild/tools/run.py | 17 ++++++++++++++--- test/framework/run.py | 21 ++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index e525f8f95a..a2854ad8fb 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -181,6 +181,19 @@ def cache_aware_func(cmd, *args, **kwargs): run_shell_cmd_cache = run_cmd_cache +def fileprefix_from_cmd(cmd, allowed_chars=False): + """ + Simplify the cmd to only the allowed_chars we want in a filename + + :param cmd: the cmd (string) + :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits) + """ + if not allowed_chars: + allowed_chars = string.ascii_letters + string.digits + + return ''.join([c for c in cmd if c in allowed_chars]) + + @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, @@ -244,9 +257,7 @@ def to_cmd_str(cmd): if output_file: toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') os.makedirs(toptmpdir, exist_ok=True) - # restrict the allowed characters in the name of the output_file - allowed_chars = string.ascii_letters + string.digits - cmd_name = ''.join([c for c in os.path.basename(cmd_str.split(' ')[0]) if c in allowed_chars]) + cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') diff --git a/test/framework/run.py b/test/framework/run.py index db74940aec..22fd2a86ba 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -35,6 +35,7 @@ import os import re import signal +import string import stat import subprocess import sys @@ -51,7 +52,7 @@ from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors -from easybuild.tools.run import complete_cmd, get_output_from_process, parse_log_for_error +from easybuild.tools.run import complete_cmd, fileprefix_from_cmd, get_output_from_process, parse_log_for_error from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN @@ -193,6 +194,24 @@ def test_run_shell_cmd_basic(self): self.assertTrue(isinstance(res.output, str)) self.assertTrue(res.work_dir and isinstance(res.work_dir, str)) + def test_fileprefix_from_cmd(self): + """test simplifications from fileprefix_from_cmd.""" + cmds = { + 'abd123': 'abd123', + 'ab"a': 'aba', + 'a{:$:S@"a': 'aSa', + } + for cmd, expected_simplification in cmds.items(): + self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) + + cmds = { + 'abd123': 'abd', + 'ab"a': 'aba', + '0a{:$:2@"a': 'aa', + } + for cmd, expected_simplification in cmds.items(): + self.assertEqual(fileprefix_from_cmd(cmd, allowed_chars=string.ascii_letters), expected_simplification) + def test_run_cmd_log(self): """Test logging of executed commands.""" fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') From 8fac144b1ff8960b788b60a883cfce1e85ef3a16 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 16 Feb 2024 10:24:36 +0100 Subject: [PATCH 094/430] fix description of `backup-modules` The documentation is wrong as it doesn't apply to `--module-only` but rather to `--rebuild` or `--force`. Also the auto-enabling is not mentioned. --- easybuild/tools/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 554775be20..0ea04c7672 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -357,7 +357,8 @@ def override_options(self): None, 'store_true', False), 'allow-use-as-root-and-accept-consequences': ("Allow using of EasyBuild as root (NOT RECOMMENDED!)", None, 'store_true', False), - 'backup-modules': ("Back up an existing module file, if any. Only works when using --module-only", + 'backup-modules': ("Back up an existing module file, if any. " + "Auto-enabled when using --module-only or --skip", None, 'store_true', None), # default None to allow auto-enabling if not disabled 'backup-patched-files': ("Create a backup (*.orig) file when applying a patch", None, 'store_true', False), From 6c000d43c26a49f3b714e7faf6c1f0fa1ef253c6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 17 Feb 2024 13:03:53 +0000 Subject: [PATCH 095/430] enhance download instructions --- easybuild/framework/easyblock.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a684b74347..aac45552e7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -53,6 +53,7 @@ import time import traceback from datetime import datetime +from textwrap import indent import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain @@ -952,7 +953,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if download_instructions is None: download_instructions = self.cfg['download_instructions'] if download_instructions is not None and download_instructions != "": - msg = "\nDownload instructions:\n\n" + download_instructions + '\n' + msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n' + msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths()) print_msg(msg, prefix=False, stderr=True) error_msg += "please follow the download instructions above, and make the file available " error_msg += "in the active source path (%s)" % ':'.join(source_paths()) From 3e8632f7ab414ecbc52ac59eee6bcfb31133cdcb Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 17 Feb 2024 13:09:40 +0000 Subject: [PATCH 096/430] update test --- test/framework/easyblock.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 64c554ca89..93d9fed793 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1582,7 +1582,8 @@ def test_download_instructions(self): self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) stderr = self.get_stderr().strip() self.mock_stderr(False) - self.assertIn("Download instructions:\n\nManual download from example.com required", stderr) + self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr) + self.assertIn("Make the files available in the active source path", stderr) # create dummy source file write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '') @@ -1596,7 +1597,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nManual download from example.com required", stderr) + self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr) + self.assertIn("Make the files available in the active source path", stderr) # wipe top-level download instructions, try again self.contents = self.contents.replace(download_instructions, '') @@ -1625,7 +1627,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr) + self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr) + self.assertIn("Make the files available in the active source path", stderr) # download instructions should also be printed if 'source_tmpl' is used to specify extension sources self.contents = self.contents.replace(sources, "'source_tmpl': SOURCE_TAR_GZ,") @@ -1638,7 +1641,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr) + self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr) + self.assertIn("Make the files available in the active source path", stderr) # create dummy source file for extension write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '') From b3092bfc9fec66ad7fa4d1be5b24ed2997bb4414 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 00:32:06 +0100 Subject: [PATCH 097/430] add unit tests for Extension.install_extension_substep --- test/framework/easyblock.py | 52 +++++++++++++++++++ .../generic/childcustomdummyextension.py | 36 +++++++++++++ .../generic/childdeprecateddummyextension.py | 36 +++++++++++++ .../generic/customdummyextension.py | 40 ++++++++++++++ .../generic/deprecateddummyextension.py | 40 ++++++++++++++ 5 files changed, 204 insertions(+) create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e29d900497..ccd72aa0e2 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1038,6 +1038,58 @@ def test_extensions_step(self): eb.close_log() os.remove(eb.logfile) + def test_extensions_step_deprecations(self): + """Test extension install with deprecated substeps.""" + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + 'exts_defaultclass = "DummyExtension"', + 'exts_list = ["ext1"]', + 'exts_list = [', + ' "dummy_ext",', + ' ("custom_ext", "0.0", {"easyblock": "CustomDummyExtension"}),', + ' ("deprec_ext", "0.0", {"easyblock": "DeprecatedDummyExtension"}),', + ' ("childcustom_ext", "0.0", {"easyblock": "ChildCustomDummyExtension"}),', + ' ("childdeprec_ext", "0.0", {"easyblock": "ChildDeprecatedDummyExtension"}),', + ']', + ]) + write_file(test_ec, test_ec_txt) + ec = process_easyconfig(test_ec)[0] + eb = get_easyblock_instance(ec) + eb.prepare_for_extensions() + eb.init_ext_instances() + + # Default DummyExtension without deprecated or custom install substeps + ext = eb.ext_instances[0] + self.assertEqual(ext.__class__.__name__, "DummyExtension") + self.assertEqual(ext.install_extension_substep("install_extension"), None) + # CustomDummyExtension + ext = eb.ext_instances[1] + self.assertEqual(ext.__class__.__name__, "CustomDummyExtension") + expected_return = "Extension installed with install_extension()" + self.assertEqual(ext.install_extension_substep("install_extension"), expected_return) + # DeprecatedDummyExtension + ext = eb.ext_instances[2] + self.assertEqual(ext.__class__.__name__, "DeprecatedDummyExtension") + error_msg = r"DEPRECATED \(since v6.0\).*use install_extension\(\) instead.*" + self.assertErrorRegex(EasyBuildError, error_msg, ext.install_extension_substep, "install_extension") + # ChildCustomDummyExtension + ext = eb.ext_instances[3] + self.assertEqual(ext.__class__.__name__, "ChildCustomDummyExtension") + expected_return = "Extension installed with install_extension()" + self.assertEqual(ext.install_extension_substep("install_extension"), expected_return) + # ChildDeprecatedDummyExtension + ext = eb.ext_instances[4] + self.assertEqual(ext.__class__.__name__, "ChildDeprecatedDummyExtension") + error_msg = r"DEPRECATED \(since v6.0\).*use install_extension\(\) instead.*" + self.assertErrorRegex(EasyBuildError, error_msg, ext.install_extension_substep, "install_extension") + def test_init_extensions(self): """Test creating extension instances.""" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py new file mode 100644 index 0000000000..43b2ef6cbe --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py @@ -0,0 +1,36 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Test EasyBlocks building and installing dummy extensions with customized methods + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.easyblocks.generic.customdummyextension import CustomDummyExtension + + +class ChildCustomDummyExtension(CustomDummyExtension): + """Extension EasyBlock inheriting customized install step""" + diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py new file mode 100644 index 0000000000..327c64118d --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py @@ -0,0 +1,36 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Test EasyBlocks building and installing dummy extensions with deprecated methods + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.easyblocks.generic.deprecateddummyextension import DeprecatedDummyExtension + + +class ChildDeprecatedDummyExtension(DeprecatedDummyExtension): + """Extension EasyBlock inheriting deprecated install step""" + diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py new file mode 100644 index 0000000000..002d2e0008 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py @@ -0,0 +1,40 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Test EasyBlocks building and installing dummy extensions with customized methods + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.easyblocks.generic.dummyextension import DummyExtension + + +class CustomDummyExtension(DummyExtension): + """Extension EasyBlock with customized install step""" + + def install_extension(self): + + return "Extension installed with install_extension()" + diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py new file mode 100644 index 0000000000..0c3c397371 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py @@ -0,0 +1,40 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Test EasyBlocks building and installing dummy extensions with deprecated methods + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.easyblocks.generic.dummyextension import DummyExtension + + +class DeprecatedDummyExtension(DummyExtension): + """Extension EasyBlock with deprecated install step""" + + def run(self): + + return "Extension installed with run()" + From 6a1b11315275cb3673c8e68363d3174e888bda06 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 00:33:35 +0100 Subject: [PATCH 098/430] fix code formatting in test easyblocks based on DummyExtension --- .../easybuild/easyblocks/generic/childcustomdummyextension.py | 1 - .../easyblocks/generic/childdeprecateddummyextension.py | 1 - .../sandbox/easybuild/easyblocks/generic/customdummyextension.py | 1 - .../easybuild/easyblocks/generic/deprecateddummyextension.py | 1 - 4 files changed, 4 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py index 43b2ef6cbe..4973f681e3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py @@ -33,4 +33,3 @@ class ChildCustomDummyExtension(CustomDummyExtension): """Extension EasyBlock inheriting customized install step""" - diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py index 327c64118d..e72b97940a 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py @@ -33,4 +33,3 @@ class ChildDeprecatedDummyExtension(DeprecatedDummyExtension): """Extension EasyBlock inheriting deprecated install step""" - diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py index 002d2e0008..da1144d980 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py @@ -37,4 +37,3 @@ class CustomDummyExtension(DummyExtension): def install_extension(self): return "Extension installed with install_extension()" - diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py index 0c3c397371..eb762da6e3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py @@ -37,4 +37,3 @@ class DeprecatedDummyExtension(DummyExtension): def run(self): return "Extension installed with run()" - From d31d260eb9e862c8c06082a59f8bb9b697f37528 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 00:52:48 +0100 Subject: [PATCH 099/430] also test pre- and post- install extension subteps with Extension.install_extension_substep --- test/framework/easyblock.py | 24 ++++++++++++------- .../generic/customdummyextension.py | 10 +++++++- .../generic/deprecateddummyextension.py | 10 +++++++- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index ccd72aa0e2..e0eb06a823 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1040,6 +1040,7 @@ def test_extensions_step(self): def test_extensions_step_deprecations(self): """Test extension install with deprecated substeps.""" + install_substeps = ["pre_install_extension", "install_extension", "post_install_extension"] test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ @@ -1068,27 +1069,32 @@ def test_extensions_step_deprecations(self): # Default DummyExtension without deprecated or custom install substeps ext = eb.ext_instances[0] self.assertEqual(ext.__class__.__name__, "DummyExtension") - self.assertEqual(ext.install_extension_substep("install_extension"), None) + for substep in install_substeps: + self.assertEqual(ext.install_extension_substep(substep), None) # CustomDummyExtension ext = eb.ext_instances[1] self.assertEqual(ext.__class__.__name__, "CustomDummyExtension") - expected_return = "Extension installed with install_extension()" - self.assertEqual(ext.install_extension_substep("install_extension"), expected_return) + for substep in install_substeps: + expected_return = f"Extension installed with custom {substep}()" + self.assertEqual(ext.install_extension_substep(substep), expected_return) # DeprecatedDummyExtension ext = eb.ext_instances[2] self.assertEqual(ext.__class__.__name__, "DeprecatedDummyExtension") - error_msg = r"DEPRECATED \(since v6.0\).*use install_extension\(\) instead.*" - self.assertErrorRegex(EasyBuildError, error_msg, ext.install_extension_substep, "install_extension") + for substep in install_substeps: + expected_error = f"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" + self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep) # ChildCustomDummyExtension ext = eb.ext_instances[3] self.assertEqual(ext.__class__.__name__, "ChildCustomDummyExtension") - expected_return = "Extension installed with install_extension()" - self.assertEqual(ext.install_extension_substep("install_extension"), expected_return) + for substep in install_substeps: + expected_return = f"Extension installed with custom {substep}()" + self.assertEqual(ext.install_extension_substep(substep), expected_return) # ChildDeprecatedDummyExtension ext = eb.ext_instances[4] self.assertEqual(ext.__class__.__name__, "ChildDeprecatedDummyExtension") - error_msg = r"DEPRECATED \(since v6.0\).*use install_extension\(\) instead.*" - self.assertErrorRegex(EasyBuildError, error_msg, ext.install_extension_substep, "install_extension") + for substep in install_substeps: + expected_error = f"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" + self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep) def test_init_extensions(self): """Test creating extension instances.""" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py index da1144d980..17a4434fe0 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py @@ -34,6 +34,14 @@ class CustomDummyExtension(DummyExtension): """Extension EasyBlock with customized install step""" + def pre_install_extension(self): + + return "Extension installed with custom pre_install_extension()" + def install_extension(self): - return "Extension installed with install_extension()" + return "Extension installed with custom install_extension()" + + def post_install_extension(self): + + return "Extension installed with custom post_install_extension()" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py index eb762da6e3..46f10d49ae 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py @@ -34,6 +34,14 @@ class DeprecatedDummyExtension(DummyExtension): """Extension EasyBlock with deprecated install step""" + def prerun(self): + + return "Extension installed with custom prerun()" + def run(self): - return "Extension installed with run()" + return "Extension installed with custom run()" + + def postrun(self): + + return "Extension installed with custom postrun()" From fe7190a65f00ddd1f9eb7677f7b26c5284e41ab4 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 00:55:57 +0100 Subject: [PATCH 100/430] use raw string formatting in EasyBlockTest.test_extensions_step_deprecations --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e0eb06a823..74b294635f 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1081,7 +1081,7 @@ def test_extensions_step_deprecations(self): ext = eb.ext_instances[2] self.assertEqual(ext.__class__.__name__, "DeprecatedDummyExtension") for substep in install_substeps: - expected_error = f"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" + expected_error = rf"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep) # ChildCustomDummyExtension ext = eb.ext_instances[3] @@ -1093,7 +1093,7 @@ def test_extensions_step_deprecations(self): ext = eb.ext_instances[4] self.assertEqual(ext.__class__.__name__, "ChildDeprecatedDummyExtension") for substep in install_substeps: - expected_error = f"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" + expected_error = rf"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep) def test_init_extensions(self): From 3b98a926e34bc8dff6a5a764c46a1fc191e6ec55 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 09:49:35 +0100 Subject: [PATCH 101/430] update docs tests with easyblocks based on DummyExtension --- test/framework/docs.py | 88 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index 70280892e4..3fbe60c23f 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -59,6 +59,10 @@ |-- EB_toy_buggy |-- ExtensionEasyBlock | |-- DummyExtension +| | |-- CustomDummyExtension +| | | |-- ChildCustomDummyExtension +| | |-- DeprecatedDummyExtension +| | | |-- ChildDeprecatedDummyExtension | |-- EB_toy | | |-- EB_toy_eula | | |-- EB_toytoy @@ -69,6 +73,10 @@ Extension |-- ExtensionEasyBlock | |-- DummyExtension +| | |-- CustomDummyExtension +| | | |-- ChildCustomDummyExtension +| | |-- DeprecatedDummyExtension +| | | |-- ChildDeprecatedDummyExtension | |-- EB_toy | | |-- EB_toy_eula | | |-- EB_toytoy @@ -91,6 +99,10 @@ |-- EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) |-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) | |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) +| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) +| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) +| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) +| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -101,6 +113,10 @@ Extension (easybuild.framework.extension) |-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) | |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) +| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) +| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) +| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) +| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -129,6 +145,16 @@ * ExtensionEasyBlock * DummyExtension + + * CustomDummyExtension + + * ChildCustomDummyExtension + + * DeprecatedDummyExtension + + * ChildDeprecatedDummyExtension + + * EB_toy * EB_toy_eula @@ -145,6 +171,16 @@ * ExtensionEasyBlock * DummyExtension + + * CustomDummyExtension + + * ChildCustomDummyExtension + + * DeprecatedDummyExtension + + * ChildDeprecatedDummyExtension + + * EB_toy * EB_toy_eula @@ -177,6 +213,16 @@ * ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + + * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + + * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + + * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + + * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + + * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) * EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) @@ -193,6 +239,16 @@ * ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + + * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + + * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + + * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + + * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + + * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) * EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) @@ -219,6 +275,10 @@ - EB_toy_buggy - ExtensionEasyBlock - DummyExtension + - CustomDummyExtension + - ChildCustomDummyExtension + - DeprecatedDummyExtension + - ChildDeprecatedDummyExtension - EB_toy - EB_toy_eula - EB_toytoy @@ -229,6 +289,10 @@ - **Extension** - ExtensionEasyBlock - DummyExtension + - CustomDummyExtension + - ChildCustomDummyExtension + - DeprecatedDummyExtension + - ChildDeprecatedDummyExtension - EB_toy - EB_toy_eula - EB_toytoy @@ -251,6 +315,10 @@ - EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -261,6 +329,10 @@ - **Extension** (easybuild.framework.extension) - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -513,8 +585,20 @@ def test_get_easyblock_classes(self): # result should correspond with test easyblocks in test/framework/sandbox/easybuild/easyblocks/generic eb_classes = get_easyblock_classes('easybuild.easyblocks.generic') eb_names = [x.__name__ for x in eb_classes] - expected = ['ConfigureMake', 'DummyExtension', 'MakeCp', 'ModuleRC', - 'PythonBundle', 'Toolchain', 'Toy_Extension', 'bar'] + expected = [ + 'ChildCustomDummyExtension', + 'ChildDeprecatedDummyExtension', + 'ConfigureMake', + 'CustomDummyExtension', + 'DeprecatedDummyExtension', + 'DummyExtension', + 'MakeCp', + 'ModuleRC', + 'PythonBundle', + 'Toolchain', + 'Toy_Extension', + 'bar', + ] self.assertEqual(sorted(eb_names), expected) def test_gen_easyblocks_overview(self): From 02810a92b940b2654c2dcba0128bde549a37d052 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 10:16:17 +0100 Subject: [PATCH 102/430] replace hardcoded paths with topdir template --- test/framework/docs.py | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index 3fbe60c23f..bcb55b279d 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -99,10 +99,10 @@ |-- EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) |-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) | |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) -| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) -| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) -| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) -| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) +| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) +| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) +| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) +| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -113,10 +113,10 @@ Extension (easybuild.framework.extension) |-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) | |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) -| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) -| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) -| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) -| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) +| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) +| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) +| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) +| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -214,13 +214,13 @@ * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) - * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) - * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) - * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) - * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) @@ -240,13 +240,13 @@ * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) - * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) - * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) - * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) - * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) @@ -315,10 +315,10 @@ - EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) - - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) - - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) - - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) - - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) + - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) + - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) + - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -329,10 +329,10 @@ - **Extension** (easybuild.framework.extension) - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) - - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) - - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) - - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) - - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) + - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) + - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) + - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) From 1ed4efe7299a304813cbe75eb899e70a5fd47870 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 10:54:57 +0100 Subject: [PATCH 103/430] disable linter checks on text blocks in the docs tests --- test/framework/docs.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index bcb55b279d..ce6fea31fc 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -80,7 +80,7 @@ | |-- EB_toy | | |-- EB_toy_eula | | |-- EB_toytoy -| |-- Toy_Extension""" +| |-- Toy_Extension""" # noqa LIST_EASYBLOCKS_DETAILED_TXT = """EasyBlock (easybuild.framework.easyblock) |-- bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) @@ -120,7 +120,7 @@ | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) -| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" +| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" # noqa LIST_EASYBLOCKS_SIMPLE_RST = """* **EasyBlock** @@ -188,7 +188,7 @@ * Toy_Extension -""" +""" # noqa LIST_EASYBLOCKS_DETAILED_RST = """* **EasyBlock** (easybuild.framework.easyblock) @@ -256,7 +256,7 @@ * Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py) -""" +""" # noqa LIST_EASYBLOCKS_SIMPLE_MD = """- **EasyBlock** - bar @@ -296,7 +296,7 @@ - EB_toy - EB_toy_eula - EB_toytoy - - Toy_Extension""" + - Toy_Extension""" # noqa LIST_EASYBLOCKS_DETAILED_MD = """- **EasyBlock** (easybuild.framework.easyblock) - bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) @@ -336,11 +336,11 @@ - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) - - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" + - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" # noqa LIST_SOFTWARE_SIMPLE_TXT = """ * GCC -* gzip""" +* gzip""" # noqa GCC_DESCR = "The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, " GCC_DESCR += "as well as libraries for these languages (libstdc++, libgcj,...)." @@ -363,7 +363,7 @@ * gzip v1.4: GCC/4.6.3, system * gzip v1.5: foss/2018a, intel/2018a -""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa LIST_SOFTWARE_SIMPLE_RST = """List of supported software ========================== @@ -379,7 +379,7 @@ --- * GCC -* gzip""" +* gzip""" # noqa LIST_SOFTWARE_DETAILED_RST = """List of supported software ========================== @@ -429,7 +429,7 @@ ``1.4`` ``GCC/4.6.3``, ``system`` ``1.5`` ``foss/2018a``, ``intel/2018a`` ======= =============================== -""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa LIST_SOFTWARE_SIMPLE_MD = """# List of supported software @@ -441,7 +441,7 @@ ## G * GCC -* gzip""" +* gzip""" # noqa LIST_SOFTWARE_DETAILED_MD = """# List of supported software @@ -475,7 +475,7 @@ version|toolchain -------|------------------------------- ``1.4``|``GCC/4.6.3``, ``system`` -``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa LIST_SOFTWARE_SIMPLE_MD = """# List of supported software @@ -487,7 +487,7 @@ ## G * GCC -* gzip""" +* gzip""" # noqa LIST_SOFTWARE_DETAILED_MD = """# List of supported software @@ -521,7 +521,7 @@ version|toolchain -------|------------------------------- ``1.4``|``GCC/4.6.3``, ``system`` -``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa LIST_SOFTWARE_SIMPLE_JSON = """[ { @@ -530,7 +530,7 @@ { "name": "gzip" } -]""" +]""" # noqa LIST_SOFTWARE_DETAILED_JSON = """[ { @@ -573,7 +573,7 @@ "version": "1.5", "versionsuffix": "" } -]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa class DocsTest(EnhancedTestCase): From dd2da2139aa601f857ba2f8047931982a555a6b4 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 11:12:54 +0100 Subject: [PATCH 104/430] update options tests with easyblocks based on DummyExtension --- test/framework/options.py | 71 ++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index ca8a582365..820e4a9c56 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -923,37 +923,46 @@ def test_000_list_easyblocks(self): logtxt = read_file(self.logfile) expected = '\n'.join([ - r'EasyBlock', - r'\|-- bar', - r'\|-- ConfigureMake', - r'\| \|-- MakeCp', - r'\|-- EB_EasyBuildMeta', - r'\|-- EB_FFTW', - r'\|-- EB_foo', - r'\| \|-- EB_foofoo', - r'\|-- EB_GCC', - r'\|-- EB_HPL', - r'\|-- EB_libtoy', - r'\|-- EB_OpenBLAS', - r'\|-- EB_OpenMPI', - r'\|-- EB_ScaLAPACK', - r'\|-- EB_toy_buggy', - r'\|-- ExtensionEasyBlock', - r'\| \|-- DummyExtension', - r'\| \|-- EB_toy', - r'\| \| \|-- EB_toy_eula', - r'\| \| \|-- EB_toytoy', - r'\| \|-- Toy_Extension', - r'\|-- ModuleRC', - r'\|-- PythonBundle', - r'\|-- Toolchain', - r'Extension', - r'\|-- ExtensionEasyBlock', - r'\| \|-- DummyExtension', - r'\| \|-- EB_toy', - r'\| \| \|-- EB_toy_eula', - r'\| \| \|-- EB_toytoy', - r'\| \|-- Toy_Extension', + "EasyBlock", + "|-- bar", + "|-- ConfigureMake", + "| |-- MakeCp", + "|-- EB_EasyBuildMeta", + "|-- EB_FFTW", + "|-- EB_foo", + "| |-- EB_foofoo", + "|-- EB_GCC", + "|-- EB_HPL", + "|-- EB_libtoy", + "|-- EB_OpenBLAS", + "|-- EB_OpenMPI", + "|-- EB_ScaLAPACK", + "|-- EB_toy_buggy", + "|-- ExtensionEasyBlock", + "| |-- DummyExtension", + "| | |-- CustomDummyExtension", + "| | | |-- ChildCustomDummyExtension", + "| | |-- DeprecatedDummyExtension", + "| | | |-- ChildDeprecatedDummyExtension", + "| |-- EB_toy", + "| | |-- EB_toy_eula", + "| | |-- EB_toytoy", + "| |-- Toy_Extension", + "|-- ModuleRC", + "|-- PythonBundle", + "|-- Toolchain", + "Extension", + "|-- ExtensionEasyBlock", + "| |-- DummyExtension", + "| | |-- CustomDummyExtension", + "| | | |-- ChildCustomDummyExtension", + "| | |-- DeprecatedDummyExtension", + "| | | |-- ChildDeprecatedDummyExtension", + "| |-- EB_toy", + "| | |-- EB_toy_eula", + "| | |-- EB_toytoy", + "| |-- Toy_Extension", + "", ]) regex = re.compile(expected, re.M) self.assertTrue(regex.search(logtxt), "Pattern '%s' found in: %s" % (regex.pattern, logtxt)) From ca3828bc1c1a16f29287e62b11c6a705aa86aceb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Feb 2024 14:34:09 +0100 Subject: [PATCH 105/430] refactor EasyBlock.skip_extensions_parallel --- easybuild/framework/easyblock.py | 33 ++++++++++---------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e2d84c162e..64aa182dc7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -41,6 +41,7 @@ * Davide Vanzo (Vanderbilt University) * Caspar van Leeuwen (SURF) """ +import concurrent import copy import glob import inspect @@ -1818,22 +1819,20 @@ def skip_extensions_parallel(self, exts_filter): self.log.experimental("Skipping installed extensions in parallel") print_msg("skipping installed extensions (in parallel)", log=self.log) - thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) - - async_shell_cmd_tasks = [] installed_exts_ids = [] - exts_queue = list(enumerate(self.ext_instances[:])) checked_exts_cnt = 0 exts_cnt = len(self.ext_instances) - done_tasks = [] + cmds = [resolve_exts_filter_template(exts_filter, ext) for ext in self.ext_instances] - # asynchronously run checks to see whether extensions are already installed - while exts_queue or async_shell_cmd_tasks: + with ThreadPoolExecutor(max_workers=self.cfg['parallel']) as thread_pool: - # first handle completed checks - for task in done_tasks: - async_shell_cmd_tasks.remove(task) - res = task.result() + # list of command to run asynchronously + async_cmds = [thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, fail_on_error=False, + asynchronous=True, task_id=idx) for (idx, (cmd, stdin)) in enumerate(cmds)] + + # process result of commands as they have completed running + for done_task in concurrent.futures.as_completed(async_cmds): + res = done_task.result() idx = res.task_id ext_name = self.ext_instances[idx].name self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") @@ -1846,16 +1845,6 @@ def skip_extensions_parallel(self, exts_filter): exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) self.update_exts_progress_bar(exts_pbar_label) - # start additional checks asynchronously - while exts_queue: - idx, ext = exts_queue.pop(0) - cmd, stdin = resolve_exts_filter_template(exts_filter, ext) - task = thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, - fail_on_error=False, asynchronous=True, task_id=idx) - async_shell_cmd_tasks.append(task) - - (done_tasks, _) = wait(async_shell_cmd_tasks, timeout=1, return_when=FIRST_COMPLETED) - # compose new list of extensions, skip over the ones that are already installed; # note: original order in extensions list should be preserved! retained_ext_instances = [] @@ -1866,8 +1855,6 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances - thread_pool.shutdown() - def install_extensions(self, install=True): """ Install extensions. From f7c0ff2295789736557861ede1b82172e40f6ab7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Feb 2024 15:51:16 +0100 Subject: [PATCH 106/430] clean up unused imports from concurrent.futures --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 64aa182dc7..4dd4bcfd81 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -52,7 +52,7 @@ import tempfile import time import traceback -from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from concurrent.futures import ThreadPoolExecutor from datetime import datetime import easybuild.tools.environment as env From 4ff3663e6dbb9aa18a875acd00784559df2352cb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 13 Feb 2024 21:26:43 +0100 Subject: [PATCH 107/430] implement basic support for qa_patterns in run_shell_cmd --- easybuild/tools/run.py | 36 +++++++++++++++++++++++++++++++----- test/framework/run.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 95391f3821..da259a8721 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -36,6 +36,7 @@ * Ward Poelmans (Ghent University) """ import contextlib +import fcntl import functools import inspect import os @@ -231,7 +232,7 @@ def to_cmd_str(cmd): return cmd_str # temporarily raise a NotImplementedError until all options are implemented - if qa_patterns or qa_wait_patterns: + if qa_wait_patterns: raise NotImplementedError if work_dir is None: @@ -315,23 +316,48 @@ def to_cmd_str(cmd): if stdin: stdin = stdin.encode() - if stream_output: + if stream_output or qa_patterns: + + if qa_patterns: + # make stdout, stderr, stdin non-blocking files + channels = [proc.stdout, proc.stdin] + if split_stderr: + channels += proc.stderr + for channel in channels: + fd = channel.fileno() + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + if stdin: proc.stdin.write(stdin) exit_code = None stdout, stderr = b'', b'' + # collect output piece-wise, while checking for questions to answer (if qa_patterns is provided) while exit_code is None: - exit_code = proc.poll() # use small read size (128 bytes) when streaming output, to make it stream more fluently # -1 means reading until EOF read_size = 128 if exit_code is None else -1 - stdout += proc.stdout.read(read_size) + exit_code = proc.poll() + + more_stdout = proc.stdout.read1(read_size) or b'' + stdout += more_stdout + + # note: we assume that there won't be any questions in stderr output if split_stderr: - stderr += proc.stderr.read(read_size) + stderr += proc.stderr.read1(read_size) or b'' + + # only consider answering questions if there's new output beyond additional whitespace + if more_stdout.strip() and qa_patterns: + for question, answer in qa_patterns: + question += '[\s\n]*$' + regex = re.compile(question.encode()) + if regex.search(stdout): + answer += '\n' + x= os.write(proc.stdin.fileno(), answer.encode()) else: (stdout, stderr) = proc.communicate(input=stdin) diff --git a/test/framework/run.py b/test/framework/run.py index c86d8635ed..887a9b0030 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -750,6 +750,40 @@ def test_run_cmd_qa(self): self.assertTrue(out.startswith("question\nanswer\nfoo ")) self.assertTrue(out.endswith('bar')) + def test_run_shell_cmd_qa(self): + """Basic test for Q&A support in run_shell_cmd function.""" + + cmd = '; '.join([ + "echo question1", + "read x", + "echo $x", + "echo question2", + "read y", + "echo $y", + ]) + qa = [ + ('question1', 'answer1'), + ('question2', 'answer2'), + ] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.output, "question1\nanswer1\nquestion2\nanswer2\n") + # no reason echo hello could fail + self.assertEqual(res.exit_code, 0) + + # test running command that emits non-UTF8 characters + # this is constructed to reproduce errors like: + # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 + test_file = os.path.join(self.test_prefix, 'foo.txt') + write_file(test_file, b"foo \xe2 bar") + cmd += "; cat %s" % test_file + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.exit_code, 0) + self.assertTrue(res.output.startswith("question1\nanswer1\nquestion2\nanswer2\nfoo ")) + self.assertTrue(res.output.endswith('bar')) + def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" From 1cce87361f6465f7524b10e5d3ac9887e93dae03 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 20:51:12 +0100 Subject: [PATCH 108/430] add support for qa_timeout in run_shell_cmd + check type of qa_patterns value --- easybuild/tools/run.py | 23 ++++++++++++++++++++- test/framework/run.py | 47 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index da259a8721..538c952cfb 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -192,7 +192,7 @@ def cache_aware_func(cmd, *args, **kwargs): def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True, - qa_patterns=None, qa_wait_patterns=None): + qa_patterns=None, qa_wait_patterns=None, qa_timeout=10000): """ Run specified (interactive) shell command, and capture output + exit code. @@ -213,6 +213,8 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions and number of iterations to allow these patterns to match with end out command output + :param qa_timeout: amount of milliseconds to wait until more output is produced when there is no matching question + :return: Named tuple with: - output: command output, stdout+stderr combined if split_stderr is disabled, only stdout otherwise - exit_code: exit code of command (integer) @@ -231,6 +233,11 @@ def to_cmd_str(cmd): return cmd_str + # make sure that qa_patterns is a list of 2-tuples (not a dict, or something else) + if qa_patterns: + if not isinstance(qa_patterns, list) or any(not isinstance(x, tuple) or len(x) != 2 for x in qa_patterns): + raise EasyBuildError("qa_patterns passed to run_shell_cmd should be a list of 2-tuples!") + # temporarily raise a NotImplementedError until all options are implemented if qa_wait_patterns: raise NotImplementedError @@ -333,6 +340,8 @@ def to_cmd_str(cmd): exit_code = None stdout, stderr = b'', b'' + check_interval_secs = 0.001 + time_no_match = 0 # collect output piece-wise, while checking for questions to answer (if qa_patterns is provided) while exit_code is None: @@ -351,6 +360,7 @@ def to_cmd_str(cmd): stderr += proc.stderr.read1(read_size) or b'' # only consider answering questions if there's new output beyond additional whitespace + hit = False if more_stdout.strip() and qa_patterns: for question, answer in qa_patterns: question += '[\s\n]*$' @@ -358,6 +368,17 @@ def to_cmd_str(cmd): if regex.search(stdout): answer += '\n' x= os.write(proc.stdin.fileno(), answer.encode()) + hit = True + + time.sleep(check_interval_secs) + if hit: + time_no_match = 0 + else: + time_no_match += check_interval_secs + if time_no_match >= qa_timeout: + error_msg = "No matching questions found for current command output, " + error_msg += f"giving up after {qa_timeout} seconds!" + raise EasyBuildError(error_msg) else: (stdout, stderr) = proc.communicate(input=stdin) diff --git a/test/framework/run.py b/test/framework/run.py index 887a9b0030..0dd1b4888b 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -784,6 +784,21 @@ def test_run_shell_cmd_qa(self): self.assertTrue(res.output.startswith("question1\nanswer1\nquestion2\nanswer2\nfoo ")) self.assertTrue(res.output.endswith('bar')) + # check type check on qa_patterns + error_pattern = "qa_patterns passed to run_shell_cmd should be a list of 2-tuples!" + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns={'foo': 'bar'}) + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=('foo', 'bar')) + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=(('foo', 'bar'),)) + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns='foo:bar') + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=['foo:bar']) + + # validate use of qa_timeout to give up if there's no matching question for too long + cmd = "sleep 3; echo 'question'; read a; echo $a" + error_pattern = "No matching questions found for current command output, giving up after 1 seconds!" + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1) + def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" @@ -816,6 +831,38 @@ def test_run_cmd_qa_buffering(self): self.assertEqual(ec, 1) self.assertEqual(out, "Hello, I am about to exit\nERROR: I failed\n") + def test_run_shell_cmd_qa_buffering(self): + """Test whether run_shell_cmd uses unbuffered output when running interactive commands.""" + + # command that generates a lot of output before waiting for input + # note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000! + cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; ' + cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"' + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=[('Pick a number: ', '42')], qa_timeout=10) + + self.assertEqual(res.exit_code, 0) + regex = re.compile("Picked number: 42$") + self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' found in: {res.output}") + + # also test with script run as interactive command that quickly exits with non-zero exit code; + # see https://github.com/easybuilders/easybuild-framework/issues/3593 + script_txt = '\n'.join([ + "#/bin/bash", + "echo 'Hello, I am about to exit'", + "echo 'ERROR: I failed' >&2", + "exit 1", + ]) + script = os.path.join(self.test_prefix, 'test.sh') + write_file(script, script_txt) + adjust_permissions(script, stat.S_IXUSR) + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(script, qa_patterns=[], fail_on_error=False) + + self.assertEqual(res.exit_code, 1) + self.assertEqual(res.output, "Hello, I am about to exit\nERROR: I failed\n") + def test_run_cmd_qa_log_all(self): """Test run_cmd_qa with log_output enabled""" with self.mocked_stdout_stderr(): From 651da9715f3f30a038a56e4d940ec5d21a3d0928 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 20:54:34 +0100 Subject: [PATCH 109/430] fix trivial code style issues --- easybuild/tools/run.py | 4 ++-- test/framework/run.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 538c952cfb..78aeab85d8 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -363,11 +363,11 @@ def to_cmd_str(cmd): hit = False if more_stdout.strip() and qa_patterns: for question, answer in qa_patterns: - question += '[\s\n]*$' + question += r'[\s\n]*$' regex = re.compile(question.encode()) if regex.search(stdout): answer += '\n' - x= os.write(proc.stdin.fileno(), answer.encode()) + os.write(proc.stdin.fileno(), answer.encode()) hit = True time.sleep(check_interval_secs) diff --git a/test/framework/run.py b/test/framework/run.py index 0dd1b4888b..5697f39e91 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -779,7 +779,7 @@ def test_run_shell_cmd_qa(self): cmd += "; cat %s" % test_file with self.mocked_stdout_stderr(): - res = run_shell_cmd(cmd, qa_patterns=qa) + res = run_shell_cmd(cmd, qa_patterns=qa) self.assertEqual(res.exit_code, 0) self.assertTrue(res.output.startswith("question1\nanswer1\nquestion2\nanswer2\nfoo ")) self.assertTrue(res.output.endswith('bar')) From f01ef65d3cdb0bfa3a3fd80a33bbe874095dd512 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 21:01:54 +0100 Subject: [PATCH 110/430] simplify qa_timeout code in run_shell_cmd by using for-else statement --- easybuild/tools/run.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 78aeab85d8..93e4afa65d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -360,25 +360,24 @@ def to_cmd_str(cmd): stderr += proc.stderr.read1(read_size) or b'' # only consider answering questions if there's new output beyond additional whitespace - hit = False - if more_stdout.strip() and qa_patterns: + if qa_patterns: for question, answer in qa_patterns: question += r'[\s\n]*$' regex = re.compile(question.encode()) if regex.search(stdout): answer += '\n' os.write(proc.stdin.fileno(), answer.encode()) - hit = True + time_no_match = 0 + break + else: + # this will only run if the for loop above was *not* stopped by the break statement + time_no_match += check_interval_secs + if time_no_match > qa_timeout: + error_msg = "No matching questions found for current command output, " + error_msg += f"giving up after {qa_timeout} seconds!" + raise EasyBuildError(error_msg) time.sleep(check_interval_secs) - if hit: - time_no_match = 0 - else: - time_no_match += check_interval_secs - if time_no_match >= qa_timeout: - error_msg = "No matching questions found for current command output, " - error_msg += f"giving up after {qa_timeout} seconds!" - raise EasyBuildError(error_msg) else: (stdout, stderr) = proc.communicate(input=stdin) From 5899ce284ef45dce7c648ace17a2b95487e4765a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 21:24:29 +0100 Subject: [PATCH 111/430] fix default value for qa_timeout (we're counting seconds, not milliseconds) --- easybuild/tools/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 93e4afa65d..1ffcd9054d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -192,7 +192,7 @@ def cache_aware_func(cmd, *args, **kwargs): def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True, - qa_patterns=None, qa_wait_patterns=None, qa_timeout=10000): + qa_patterns=None, qa_wait_patterns=None, qa_timeout=100): """ Run specified (interactive) shell command, and capture output + exit code. @@ -213,7 +213,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions and number of iterations to allow these patterns to match with end out command output - :param qa_timeout: amount of milliseconds to wait until more output is produced when there is no matching question + :param qa_timeout: amount of seconds to wait until more output is produced when there is no matching question :return: Named tuple with: - output: command output, stdout+stderr combined if split_stderr is disabled, only stdout otherwise From cfb7f3ad9edd9cdbc7f8515a5d5157cdfe519257 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 21:39:04 +0100 Subject: [PATCH 112/430] fix trace message for interactive commands run with run_shell_cmd --- easybuild/tools/run.py | 14 +++++++----- test/framework/run.py | 48 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 1ffcd9054d..5a8bc298ff 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -277,7 +277,8 @@ def to_cmd_str(cmd): if not in_dry_run and build_option('extended_dry_run'): if not hidden or verbose_dry_run: silent = build_option('silent') - msg = f" running shell command \"{cmd_str}\"\n" + interactive = 'interactive ' if qa_patterns else '' + msg = f" running {interactive}shell command \"{cmd_str}\"\n" msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) @@ -286,7 +287,8 @@ def to_cmd_str(cmd): start_time = datetime.now() if not hidden: - _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id) + _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, + interactive=bool(qa_patterns)) if stream_output: print_msg(f"(streaming) output for command '{cmd_str}':") @@ -430,7 +432,7 @@ def to_cmd_str(cmd): return res -def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id): +def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, interactive=False): """ Helper function to construct and print trace message for command being run @@ -441,13 +443,15 @@ def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thr :param cmd_out_fp: path to output file for command :param cmd_err_fp: path to errors/warnings output file for command :param thread_id: thread ID (None when not running shell command asynchronously) + :param interactive: boolean indicating whether it is an interactive command, or not """ start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') + interactive = 'interactive ' if interactive else '' if thread_id: - run_cmd_msg = f"running shell command (asynchronously, thread ID: {thread_id}):" + run_cmd_msg = f"running {interactive}shell command (asynchronously, thread ID: {thread_id}):" else: - run_cmd_msg = "running shell command:" + run_cmd_msg = f"running {interactive}shell command:" lines = [ run_cmd_msg, diff --git a/test/framework/run.py b/test/framework/run.py index 5697f39e91..669b8fa073 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -876,13 +876,22 @@ def test_run_cmd_qa_log_all(self): extra_pref = "# output for interactive command: echo 'n: '; read n; seq 1 $n\n\n" self.assertEqual(run_cmd_log_txt, extra_pref + "n: \n1\n2\n3\n4\n5\n") + def test_run_shell_cmd_qa_log(self): + """Test temporary log file for run_shell_cmd with qa_patterns""" + with self.mocked_stdout_stderr(): + res = run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n:', '5')]) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "n: \n1\n2\n3\n4\n5\n") + + run_cmd_logs = glob.glob(os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output', 'echo-*', 'out.txt')) + self.assertEqual(len(run_cmd_logs), 1) + run_cmd_log_txt = read_file(run_cmd_logs[0]) + self.assertEqual(run_cmd_log_txt, "n: \n1\n2\n3\n4\n5\n") + def test_run_cmd_qa_trace(self): """Test run_cmd under --trace""" - # replace log.experimental with log.warning to allow experimental code - easybuild.tools.utilities._log.experimental = easybuild.tools.utilities._log.warning - - init_config(build_options={'trace': True}) + # --trace is enabled by default self.mock_stdout(True) self.mock_stderr(True) (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}) @@ -910,6 +919,37 @@ def test_run_cmd_qa_trace(self): self.assertEqual(stdout, '') self.assertEqual(stderr, '') + def test_run_shell_cmd_qa_trace(self): + """Test run_shell_cmd with qa_patterns under --trace""" + + # --trace is enabled by default + self.mock_stdout(True) + self.mock_stderr(True) + run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n: ', '5')]) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertEqual(stderr, '') + pattern = r"^ >> running interactive shell command:\n" + pattern += r"\techo \'n: \'; read n; seq 1 \$n\n" + pattern += r"\t\[started at: .*\]\n" + pattern += r"\t\[working dir: .*\]\n" + pattern += r"\t\[output saved to .*\]\n" + pattern += r' >> command completed: exit 0, ran in .*' + self.assertTrue(re.search(pattern, stdout), "Pattern '%s' found in: %s" % (pattern, stdout)) + + # trace output can be disabled on a per-command basis + self.mock_stdout(True) + self.mock_stderr(True) + run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n: ', '5')], hidden=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertEqual(stdout, '') + self.assertEqual(stderr, '') + def test_run_cmd_qa_answers(self): """Test providing list of answers in run_cmd_qa.""" cmd = "echo question; read x; echo $x; " * 2 From 4b63b913ef88b402036757dc2524b91f3b866bb4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Feb 2024 16:46:51 +0100 Subject: [PATCH 113/430] make sure that *all* stdout/stderr output is read when streaming output or running interactive commands --- easybuild/tools/run.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 5a8bc298ff..09a699ca80 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -352,8 +352,6 @@ def to_cmd_str(cmd): # -1 means reading until EOF read_size = 128 if exit_code is None else -1 - exit_code = proc.poll() - more_stdout = proc.stdout.read1(read_size) or b'' stdout += more_stdout @@ -380,6 +378,12 @@ def to_cmd_str(cmd): raise EasyBuildError(error_msg) time.sleep(check_interval_secs) + + exit_code = proc.poll() + + stdout += proc.stdout.read() + if split_stderr: + stderr += proc.stderr.read() else: (stdout, stderr) = proc.communicate(input=stdin) From cb84bd2ba1610419e03f869053d3bd158e5ada1a Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:08:58 +0000 Subject: [PATCH 114/430] extra cases --- easybuild/tools/run.py | 4 ++-- test/framework/run.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index a2854ad8fb..6a568a9f70 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -186,10 +186,10 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): Simplify the cmd to only the allowed_chars we want in a filename :param cmd: the cmd (string) - :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits) + :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits + "_-") """ if not allowed_chars: - allowed_chars = string.ascii_letters + string.digits + allowed_chars = f"{string.ascii_letters}{string.digits}_-" return ''.join([c for c in cmd if c in allowed_chars]) diff --git a/test/framework/run.py b/test/framework/run.py index 22fd2a86ba..386a9cf7f6 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -200,6 +200,8 @@ def test_fileprefix_from_cmd(self): 'abd123': 'abd123', 'ab"a': 'aba', 'a{:$:S@"a': 'aSa', + 'cmd-with-dash': 'cmd-with-dash', + 'cmd_with_underscore'. 'cmd_with_underscore', } for cmd, expected_simplification in cmds.items(): self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) From a309aaea65a335abee90f4a8df09907c92eaed20 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:09:40 +0000 Subject: [PATCH 115/430] typo --- test/framework/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index 386a9cf7f6..e63e4e219c 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -201,7 +201,7 @@ def test_fileprefix_from_cmd(self): 'ab"a': 'aba', 'a{:$:S@"a': 'aSa', 'cmd-with-dash': 'cmd-with-dash', - 'cmd_with_underscore'. 'cmd_with_underscore', + 'cmd_with_underscore': 'cmd_with_underscore', } for cmd, expected_simplification in cmds.items(): self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) From d57c251c8145ff080529038f00505951eaee6983 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:34:10 +0000 Subject: [PATCH 116/430] replace `'` with `"` for `printf` to have bash replace a variable --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ec98f9df03..c79b515e42 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -178,7 +178,7 @@ jobs: echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool" continue fi - printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n' + printf "\n\n=====================> Using $module_syntax module syntax <=====================\n\n" export EASYBUILD_MODULE_SYNTAX="${module_syntax}" export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}" From 560d395a1d8ab4ef2366543b642d15ec548c1c1a Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 16:13:41 +0000 Subject: [PATCH 117/430] re-order test_cuda_compute_capabilities {pre,}{config,build,install}opts --- test/framework/easyconfig.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 11b551e55c..1ce8d1facc 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4438,31 +4438,31 @@ def test_cuda_compute_capabilities(self): description = 'test' toolchain = SYSTEM cuda_compute_capabilities = ['5.1', '7.0', '7.1'] - installopts = '%(cuda_compute_capabilities)s' - preinstallopts = '%(cuda_cc_space_sep)s' - prebuildopts = '%(cuda_cc_semicolon_sep)s' - configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' + configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' + prebuildopts = '%(cuda_cc_semicolon_sep)s' + preinstallopts = '%(cuda_cc_space_sep)s' + installopts = '%(cuda_compute_capabilities)s' """) self.prep() ec = EasyConfig(self.eb_file) - self.assertEqual(ec['installopts'], '5.1,7.0,7.1') - self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') - self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"') self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' 'space="sm_51 sm_70 sm_71"') - self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"') + self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') + self.assertEqual(ec['installopts'], '5.1,7.0,7.1') # build options overwrite it init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']}) ec = EasyConfig(self.eb_file) - self.assertEqual(ec['installopts'], '4.2,6.3') - self.assertEqual(ec['preinstallopts'], '4.2 6.3') - self.assertEqual(ec['prebuildopts'], '4.2;6.3') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' 'space="sm_42 sm_63"') - self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') + self.assertEqual(ec['prebuildopts'], '4.2;6.3') + self.assertEqual(ec['preinstallopts'], '4.2 6.3') + self.assertEqual(ec['installopts'], '4.2,6.3') def test_det_copy_ec_specs(self): """Test det_copy_ec_specs function.""" From aff74821c1cc9b3e69065d370e4bd64318162a26 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 16:37:45 +0000 Subject: [PATCH 118/430] add support for cuda compute capabilities templates in the form of: cuda_int_comma_sep: 70,75 80 cuda_int_semicolon_sep: 70;75;80 cuda_int_space_sep: 70 75 80 --- easybuild/framework/easyconfig/templates.py | 7 +++++++ test/framework/easyconfig.py | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index fcefe4de55..0d8a4d5cf3 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -97,6 +97,9 @@ ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), + ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"), + ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"), + ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"), ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), ] @@ -365,6 +368,10 @@ def template_constant_dict(config, ignore=None, toolchain=None): template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities) + int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities] + template_values['cuda_int_comma_sep'] = ','.join(int_values) + template_values['cuda_int_space_sep'] = ' '.join(int_values) + template_values['cuda_int_semicolon_sep'] = ';'.join(int_values) sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 1ce8d1facc..399ce3afe8 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4441,6 +4441,8 @@ def test_cuda_compute_capabilities(self): preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" + 'semi="%(cuda_int_semicolon_sep)s"') preinstallopts = '%(cuda_cc_space_sep)s' installopts = '%(cuda_compute_capabilities)s' """) @@ -4451,6 +4453,9 @@ def test_cuda_compute_capabilities(self): self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' 'space="sm_51 sm_70 sm_71"') self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['buildopts'], 'comma="51,70,71" ' + 'space="51 70 71" ' + 'semi="51;70;71"') self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') self.assertEqual(ec['installopts'], '5.1,7.0,7.1') @@ -4460,6 +4465,9 @@ def test_cuda_compute_capabilities(self): self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' 'space="sm_42 sm_63"') + self.assertEqual(ec['buildopts'], 'comma="42,63" ' + 'space="42 63" ' + 'semi="42;63"') self.assertEqual(ec['prebuildopts'], '4.2;6.3') self.assertEqual(ec['preinstallopts'], '4.2 6.3') self.assertEqual(ec['installopts'], '4.2,6.3') @@ -4725,6 +4733,9 @@ def test_get_cuda_cc_template_value(self): 'cuda_compute_capabilities': '6.5,7.0', 'cuda_cc_space_sep': '6.5 7.0', 'cuda_cc_semicolon_sep': '6.5;7.0', + 'cuda_int_comma_sep': '65,70', + 'cuda_int_space_sep': '65 70', + 'cuda_int_semicolon_sep': '65;70', 'cuda_sm_comma_sep': 'sm_65,sm_70', 'cuda_sm_space_sep': 'sm_65 sm_70', } From f2d53d1ca449c404a246fe54bc9be897758ef39f Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 17:03:12 +0000 Subject: [PATCH 119/430] fix missing end quote --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 399ce3afe8..a61be39473 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4441,7 +4441,7 @@ def test_cuda_compute_capabilities(self): preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' - buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s"' 'semi="%(cuda_int_semicolon_sep)s"') preinstallopts = '%(cuda_cc_space_sep)s' installopts = '%(cuda_compute_capabilities)s' From b65771f3132a0dc3b9a9fae6f3b015215637ba62 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 17:41:50 +0000 Subject: [PATCH 120/430] fix missing space --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a61be39473..141e9b0f77 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4441,7 +4441,7 @@ def test_cuda_compute_capabilities(self): preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' - buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s"' + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" ' 'semi="%(cuda_int_semicolon_sep)s"') preinstallopts = '%(cuda_cc_space_sep)s' installopts = '%(cuda_compute_capabilities)s' From 3a7b3eb109c8933c68ef2a89f71391b1c243e6a7 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Fri, 23 Feb 2024 16:38:36 +0100 Subject: [PATCH 121/430] Use cp -dR instead of cp -a for shell script extraction cp -a will try to copy attributes, when the copy is from a file system type with attributes that doesn't work on the target file system this will fail --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5a2f724bc8..bc938f4210 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -161,7 +161,7 @@ # tar.Z: using compress (LZW), but can be handled with gzip so use 'z' '.tar.z': "tar xzf %(filepath)s", # shell scripts don't need to be unpacked, just copy there - '.sh': "cp -a %(filepath)s .", + '.sh': "cp -dR %(filepath)s .", } ZIPPED_PATCH_EXTS = ('.bz2', '.gz', '.xz') From 17f4b1dff887fc69e0fec73c8919335878ec4edf Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 26 Feb 2024 09:36:19 +0800 Subject: [PATCH 122/430] fix link to documentation in close_pr message --- easybuild/tools/github.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8100c8613b..96ed4d6492 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1370,8 +1370,7 @@ def close_pr(pr, motivation_msg=None): if not reopen: msg += "\nPlease don't hesitate to reopen this PR or add a comment if you feel this contribution is still " msg += "relevant.\nFor more information on our policy w.r.t. closing PRs, see " - msg += "https://easybuild.readthedocs.io/en/latest/Contributing.html" - msg += "#why-a-pull-request-may-be-closed-by-a-maintainer" + msg += "https://docs.easybuild.io/contributing/#contributing_review_process_why_pr_closed_by_maintainer" post_comment_in_issue(pr, msg, account=pr_target_account, repo=pr_target_repo, github_user=github_user) if dry_run: From 0f18f73c8207f8172efa7dda07aed9aaadd4b070 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 26 Feb 2024 07:45:37 +0100 Subject: [PATCH 123/430] tests: Adjust test of EXTRACT_CMDS for .sh files to match the change from cp -a to cp -dR --- test/framework/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3b879bef85..8ee6b2c917 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -104,7 +104,7 @@ def test_extract_cmd(self): ('test.txz', "unset TAPE; unxz test.txz --stdout | tar x"), ('test.iso', "7z x test.iso"), ('test.tar.Z', "tar xzf test.tar.Z"), - ('test.foo.bar.sh', "cp -a test.foo.bar.sh ."), + ('test.foo.bar.sh', "cp -dR test.foo.bar.sh ."), # check whether extension is stripped correct to determine name of target file # cfr. https://github.com/easybuilders/easybuild-framework/pull/3705 ('testbz2.bz2', "bunzip2 -c testbz2.bz2 > testbz2"), From 8d9e14eb6d72490c675adb8a70415a4c735cc0bc Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 29 Feb 2024 02:40:57 +0100 Subject: [PATCH 124/430] make reproducible archives only of git repos without .git dir and reset timestamps with touch --- easybuild/tools/filetools.py | 42 ++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 12fe557a60..5b7406fffa 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2540,12 +2540,12 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path) -def get_source_tarball_from_git(filename, targetdir, git_config): +def get_source_tarball_from_git(filename, target_dir, git_config): """ Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it :param filename: name of the archive to save the code to (must be .tar.gz) - :param targetdir: target directory where to save the archive to + :param target_dir: target directory where to save the archive to :param git_config: dictionary containing url, repo_name, recursive, and one of tag or commit """ # sanity check on git_config value being passed @@ -2584,8 +2584,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): raise EasyBuildError("git_config currently only supports filename ending in .tar.gz") # prepare target directory and clone repository - mkdir(targetdir, parents=True) - targetpath = os.path.join(targetdir, filename) + mkdir(target_dir, parents=True) # compose 'git clone' command, and run it if extra_config_params: @@ -2668,21 +2667,36 @@ def get_source_tarball_from_git(filename, targetdir, git_config): for cmd in cmds: run_shell_cmd(cmd, work_dir=work_dir, hidden=True, verbose_dry_run=True) - # When CentOS 7 is phased out and tar>1.28 is everywhere, replace find-sort-pipe with tar-flag - # '--sort=name' and place LC_ALL in front of tar. Also remove flags --null, --no-recursion, and - # --files-from - from the flags to tar. See https://reproducible-builds.org/docs/archives/ - tar_cmd = ['find', repo_name, '-print0', '-path \'*/.git\' -prune' if not keep_git_dir else '', '|', - 'LC_ALL=C', 'sort', '--zero-terminated', '|', - 'GZIP=--no-name', 'tar', '--create', '--file', targetpath, '--no-recursion', - '--gzip', '--mtime="1970-01-01 00:00Z"', '--owner=0', '--group=0', - '--numeric-owner', '--format=gnu', '--null', - '--no-recursion', '--files-from -'] + # Create archive + archive_path = os.path.join(target_dir, filename) + + if keep_git_dir: + # create archive of git repo including .git directory + tar_cmd = ['tar', 'cfvz', archive_path, repo_name] + else: + # create reproducible archive + # see https://reproducible-builds.org/docs/archives/ + # TODO: when CentOS 7 is phased out and tar>1.28 is everywhere, replace sort step + # in the pipe with tar-flag '--sort=name' and place LC_ALL in front of tar. + tar_cmd = [ + # print names of all files and folders excluding .git directory + 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', + # reset access and modification timestamps + '-exec', 'touch', '-t 197001010100', '{}', '\;', '|', + # sort file list + 'LC_ALL=C', 'sort', '--zero-terminated', '|', + # create tarball in GNU format with ownership reset + 'tar', '--create', '--no-recursion', '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', + '--null', '--files-from', '-', '|', + # compress tarball with gzip without original file name and timestamp + 'gzip', '--no-name', '>', archive_path + ] run_shell_cmd(' '.join(tar_cmd), work_dir=tmpdir, hidden=True, verbose_dry_run=True) # cleanup (repo_name dir does not exist in dry run mode) remove(tmpdir) - return targetpath + return archive_path def move_file(path, target_path, force_in_dry_run=False): From 46e61c85d881dc081936347930cfa44bdbd42bde Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 29 Feb 2024 02:52:25 +0100 Subject: [PATCH 125/430] update FileToolsTest.test_github_get_source_tarball_from_git tests with reproducible tar commands --- test/framework/filetools.py | 63 +++++++++++++++---------------------- 1 file changed, 25 insertions(+), 38 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3adb736586..8d823ef59c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2802,26 +2802,25 @@ def run_check(): 'git_repo': 'git@github.com:easybuilders/testrepository.git', 'test_prefix': self.test_prefix, } + reprod_tar_cmd_pattern = ( + r' running shell command "find {} -name \".git\" -prune -o -print0 -exec touch -t 197001010100 {{}} \; |' + r' LC_ALL=C sort --zero-terminated | tar --create --no-recursion --owner=0 --group=0 --numeric-owner' + r' --format=gnu --null --files-from - | gzip --no-name > %(test_prefix)s/target/test.tar.gz' + ) expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file %(test_prefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() git_config['clone_into'] = 'test123' expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', r" \(in .*/tmp.*\)", - r' running command "find test123 -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("test123"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2829,12 +2828,9 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2844,9 +2840,9 @@ def run_check(): ' running shell command "git clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", - ]) % git_repo + ]) % string_args run_check() git_config['extra_config_params'] = [ @@ -2858,21 +2854,18 @@ def run_check(): + ' clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", - ]) % git_repo + ]) % string_args run_check() del git_config['recurse_submodules'] del git_config['extra_config_params'] git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 | LC_ALL=C sort --zero-terminated | GZIP=--no-name tar' - r' --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion --gzip' - r' --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu --null --no-recursion' - r' --files-from -"', + r' running shell command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2881,28 +2874,22 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "git checkout 8456f86 && git submodule update --init --recursive"', + r' running shell command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "git checkout 8456f86"', + r' running shell command "git checkout 8456f86"', r" \(in testrepository\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2914,9 +2901,9 @@ def run_check(): r" \(in /.*\)", r' running shell command "git checkout 8456f86"', r" \(in /.*/testrepository\)", - r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in /.*\)", - ]) % git_repo + ]) % string_args run_check() # Test with real data. From 8c76561bf026831bfb7ff065a923636f20dd4b1f Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 29 Feb 2024 09:56:49 +0100 Subject: [PATCH 126/430] remove TODO about sort option in tar as it is not supported across implementations --- easybuild/tools/filetools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5b7406fffa..470e347e88 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2676,8 +2676,6 @@ def get_source_tarball_from_git(filename, target_dir, git_config): else: # create reproducible archive # see https://reproducible-builds.org/docs/archives/ - # TODO: when CentOS 7 is phased out and tar>1.28 is everywhere, replace sort step - # in the pipe with tar-flag '--sort=name' and place LC_ALL in front of tar. tar_cmd = [ # print names of all files and folders excluding .git directory 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', From 4fbf38fd314e975c112a78f7bee6dff5bb08dbc6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 29 Feb 2024 16:10:33 +0100 Subject: [PATCH 127/430] add support for --from-commit --- easybuild/framework/easyconfig/tools.py | 55 ++++++----- easybuild/tools/config.py | 3 +- easybuild/tools/github.py | 126 +++++++++++++++++++++--- easybuild/tools/options.py | 10 +- easybuild/tools/robot.py | 8 +- test/framework/easyconfig.py | 35 +++++++ test/framework/robot.py | 44 ++++++++- 7 files changed, 234 insertions(+), 47 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 73393040c6..711e14d29e 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -60,7 +60,8 @@ from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files from easybuild.tools.filetools import read_file, resolve_path, which, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO -from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data +from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit +from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.github import fetch_files_from_pr from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict @@ -310,7 +311,7 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): return paths -def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=None): +def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, from_commit=None, review_pr=None): """Obtain alternative paths for easyconfig files.""" # paths where tweaked easyconfigs will be placed, easyconfigs listed on the command line take priority and will be @@ -321,18 +322,20 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=Non tweaked_ecs_paths = (os.path.join(tmpdir, 'tweaked_easyconfigs'), os.path.join(tmpdir, 'tweaked_dep_easyconfigs')) - # paths where files touched in PRs will be downloaded to, - # which are picked up via 'pr_paths' build option in fetch_files_from_pr - pr_paths = [] + # paths where files touched in commit/PRs will be downloaded to, + # which are picked up via 'extra_ec_paths' build option in fetch_files_from_pr + extra_ec_paths = [] if from_prs: - pr_paths = from_prs[:] - if review_pr and review_pr not in pr_paths: - pr_paths.append(review_pr) + extra_ec_paths = from_prs[:] + if review_pr and review_pr not in extra_ec_paths: + extra_ec_paths.append(review_pr) + if extra_ec_paths: + extra_ec_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in extra_ec_paths] - if pr_paths: - pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in pr_paths] + if from_commit: + extra_ec_paths.append(os.path.join(tmpdir, 'files_commit_' + from_commit)) - return tweaked_ecs_paths, pr_paths + return tweaked_ecs_paths, extra_ec_paths def det_easyconfig_paths(orig_paths): @@ -346,27 +349,31 @@ def det_easyconfig_paths(orig_paths): except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + from_commit = build_option('from_commit') robot_path = build_option('robot_path') # list of specified easyconfig files ec_files = orig_paths[:] + commit_files, pr_files = [], [] if from_prs: - pr_files = [] for pr in from_prs: - # path to where easyconfig files should be downloaded is determined via 'pr_paths' build option, - # which corresponds to the list of PR paths returned by alt_easyconfig_paths + # path to where easyconfig files should be downloaded is determined + # via 'extra_ec_paths' build options, + # which corresponds to the list of commit/PR paths returned by alt_easyconfig_paths pr_files.extend(fetch_easyconfigs_from_pr(pr)) - - if ec_files: - # replace paths for specified easyconfigs that are touched in PR - for i, ec_file in enumerate(ec_files): - for pr_file in pr_files: - if ec_file == os.path.basename(pr_file): - ec_files[i] = pr_file - else: - # if no easyconfigs are specified, use all the ones touched in the PR - ec_files = [path for path in pr_files if path.endswith('.eb')] + elif from_commit: + commit_files = fetch_easyconfigs_from_commit(from_commit, files=ec_files) + + if ec_files: + # replace paths for specified easyconfigs that are touched in commit/PRs + for i, ec_file in enumerate(ec_files): + for file in commit_files + pr_files: + if ec_file == os.path.basename(file): + ec_files[i] = file + else: + # if no easyconfigs are specified, use all the ones touched in the commit/PRs + ec_files = [path for path in commit_files + pr_files if path.endswith('.eb')] filter_ecs = build_option('filter_ecs') if filter_ecs: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ee8eb5fc53..ebc4f4a7d1 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -220,6 +220,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'filter_env_vars', 'filter_rpath_sanity_libs', 'force_download', + 'from_commit', 'git_working_dirs_path', 'github_user', 'github_org', @@ -403,7 +404,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'build_specs', 'command_line', 'external_modules_metadata', - 'pr_paths', + 'extra_ec_paths', 'robot_path', 'valid_module_classes', 'valid_stops', diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 96ed4d6492..0cfc181120 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -54,7 +54,7 @@ from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option -from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_framework_files +from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_file, copy_framework_files from easybuild.tools.filetools import det_patched_files, download_file, extract_file from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen @@ -355,7 +355,7 @@ def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token= return res -def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_MAIN, path=None, github_user=None): +def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, account=GITHUB_EB_MAIN, path=None, github_user=None): """ Download entire GitHub repo as a tar.gz archive, and extract it into specified path. :param repo: repo to download @@ -364,7 +364,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M :param path: path to extract to :param github_user: name of GitHub user to use """ - if branch is None: + if branch is None and commit is None: branch = pick_default_branch(account) # make sure path exists, create it if necessary @@ -375,9 +375,16 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M path = os.path.join(path, account) mkdir(path, parents=True) - extracted_dir_name = '%s-%s' % (repo, branch) - base_name = '%s.tar.gz' % branch - latest_commit_sha = fetch_latest_commit_sha(repo, account, branch, github_user=github_user) + if commit: + extracted_dir_name = '%s-%s' % (repo, commit) + base_name = '%s.tar.gz' % commit + latest_commit_sha = commit + elif branch: + extracted_dir_name = '%s-%s' % (repo, branch) + base_name = '%s.tar.gz' % branch + latest_commit_sha = fetch_latest_commit_sha(repo, account, branch, github_user=github_user) + else: + raise EasyBuildError("Either branch or commit should be specified in download_repo") expected_path = os.path.join(path, extracted_dir_name) latest_sha_path = os.path.join(expected_path, 'latest-sha') @@ -401,11 +408,22 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M # check if extracted_path exists if not os.path.isdir(extracted_path): - raise EasyBuildError("%s should exist and contain the repo %s at branch %s", extracted_path, repo, branch) + error_msg = "%s should exist and contain the repo %s " % (extracted_path, repo) + if branch: + error_msg += "at branch " + branch + elif commit: + error_msg += "at commit " + commit + raise EasyBuildError(error_msg) write_file(latest_sha_path, latest_commit_sha, forced=True) - _log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path)) + log_msg = "Repo %s at %%s extracted into %s" % (repo, extracted_path) + if branch: + log_msg = log_msg % ('branch ' + branch) + elif commit: + log_msg = log_msg % ('commit ' + commit) + _log.debug(log_msg) + return extracted_path @@ -450,14 +468,14 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi if path is None: if github_repo == GITHUB_EASYCONFIGS_REPO: - pr_paths = build_option('pr_paths') - if pr_paths: + extra_ec_paths = build_option('extra_ec_paths') + if extra_ec_paths: # figure out directory for this specific PR (see also alt_easyconfig_paths) - cands = [p for p in pr_paths if p.endswith('files_pr%s' % pr)] + cands = [p for p in extra_ec_paths if p.endswith('files_pr%s' % pr)] if len(cands) == 1: path = cands[0] else: - raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, pr_paths) + raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, extra_ec_paths) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr) @@ -567,6 +585,90 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO) +def fetch_files_from_commit(commit, files=None, path=None, github_account=None, github_repo=None): + """ + Fetch files from a specific commit. + + If 'files' is None, all files touched in the commit are used. + """ + if github_account is None: + github_account = build_option('pr_target_account') + + if github_repo is None: + github_repo = GITHUB_EASYCONFIGS_REPO + + if path is None: + if github_repo == GITHUB_EASYCONFIGS_REPO: + extra_ec_paths = build_option('extra_ec_paths') + if extra_ec_paths: + # figure out directory for this specific commit (see also alt_easyconfig_paths) + cands = [p for p in extra_ec_paths if p.endswith('files_commit_' + commit)] + if len(cands) == 1: + path = cands[0] + else: + raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s", commit, extra_ec_paths) + + elif github_repo == GITHUB_EASYBLOCKS_REPO: + path = os.path.join(tempfile.gettempdir(), 'ebs_commit_' + commit) + else: + raise EasyBuildError("Unknown repo: %s" % github_repo) + + # if no files are specified, determine which files are touched in commit + if not files: + diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff') + diff_fn = os.path.basename(diff_url) + diff_filepath = os.path.join(path, diff_fn) + download_file(diff_fn, diff_url, diff_filepath, forced=True) + diff_txt = read_file(diff_filepath) + _log.debug("Diff for commit %s:\n%s", commit, diff_txt) + + files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True) + _log.debug("List of patched files for commit %s: %s", commit, files) + + # download tarball for specific commit + repo_commit = download_repo(repo=github_repo, commit=commit, account=github_account) + + if github_repo == GITHUB_EASYCONFIGS_REPO: + files_subdir = 'easybuild/easyconfigs/' + elif github_repo == GITHUB_EASYBLOCKS_REPO: + files_subdir = 'easybuild/easyblocks/' + + # copy specified files to directory where they're expected to be found + file_paths = [] + for file in files: + + # if only filename is specified, we need to determine the file path + if file == os.path.basename(file): + src_path = None + for (dirpath, dirnames, filenames) in os.walk(repo_commit, topdown=True): + if file in filenames: + src_path = os.path.join(dirpath, file) + break + else: + src_path = os.path.join(repo_commit, file) + + # strip of leading subdirectory like easybuild/easyconfigs/ or easybuild/easyblocks/ + # because that's what expected by robot_find_easyconfig + if file.startswith(files_subdir): + file = file[len(files_subdir):] + + # if file is found, copy it to dedicated directory; + # if not, just skip it (may be an easyconfig file in local directory); + if src_path and os.path.exists(src_path): + target_path = os.path.join(path, file) + copy_file(src_path, target_path) + file_paths.append(target_path) + else: + _log.info("File %s not found in %s, so ignoring it...", file, repo_commit) + + return file_paths + + +def fetch_easyconfigs_from_commit(commit, files=None, path=None): + """Fetch specified easyconfig files from a specific commit.""" + return fetch_files_from_commit(commit, files=files, path=path, github_repo=GITHUB_EASYCONFIGS_REPO) + + def create_gist(txt, fn, descr=None, github_user=None, github_token=None): """Create a gist with the provided text.""" diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 0ea04c7672..f99a5ac284 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -702,6 +702,7 @@ def github_options(self): 'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False), 'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), + 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', None, {'metavar': 'PR#'}), 'from-pr': ("Obtain easyconfigs from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None), 'github-user': ("GitHub username", str, 'store', None), @@ -1588,10 +1589,11 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs - tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs, - review_pr=review_pr) + tweaked_ecs_paths, extra_ec_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs, + from_commit=eb_go.options.from_commit, + review_pr=review_pr) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query - robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_paths, auto_robot=auto_robot) + robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, extra_ec_paths, auto_robot=auto_robot) log.debug("Full robot path: %s", robot_path) if not robot_path: @@ -1605,7 +1607,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r 'build_specs': build_specs, 'command_line': eb_cmd_line, 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata), - 'pr_paths': pr_paths, + 'extra_ec_paths': extra_ec_paths, 'robot_path': robot_path, 'silent': testing or new_update_opt, 'try_to_generate': try_to_generate, diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 68a4c9028b..e49f788d53 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -54,7 +54,7 @@ _log = fancylogger.getLogger('tools.robot', fname=False) -def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_paths, auto_robot=False): +def det_robot_path(robot_paths_option, tweaked_ecs_paths, extra_ec_paths, auto_robot=False): """Determine robot path.""" robot_path = robot_paths_option[:] _log.info("Using robot path(s): %s", robot_path) @@ -70,9 +70,9 @@ def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_paths, auto_robot=F _log.info("Prepended list of robot search paths with %s and appended with %s: %s", tweaked_ecs_path, tweaked_ecs_deps_path, robot_path) - if pr_paths is not None: - robot_path.extend(pr_paths) - _log.info("Extended list of robot search paths with %s: %s", pr_paths, robot_path) + if extra_ec_paths is not None: + robot_path.extend(extra_ec_paths) + _log.info("Extended list of robot search paths with %s: %s", extra_ec_paths, robot_path) return robot_path diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 905cf6c15b..2cadf46049 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -733,6 +733,41 @@ def test_tweaking(self): # cleanup os.remove(tweaked_fn) + def test_alt_easyconfig_paths(self): + """Test alt_easyconfig_paths function that collects list of additional paths for easyconfig files.""" + + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix) + self.assertEqual(tweaked_ecs_path, None) + self.assertEqual(extra_ecs_path, []) + + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) + self.assertTrue(tweaked_ecs_path) + self.assertTrue(isinstance(tweaked_ecs_path, tuple)) + self.assertEqual(len(tweaked_ecs_path), 2) + self.assertEqual(tweaked_ecs_path[0], os.path.join(self.test_prefix, 'tweaked_easyconfigs')) + self.assertEqual(tweaked_ecs_path[1], os.path.join(self.test_prefix, 'tweaked_dep_easyconfigs')) + self.assertEqual(extra_ecs_path, []) + + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123,456]) + self.assertEqual(tweaked_ecs_path, None) + self.assertTrue(extra_ecs_path) + self.assertTrue(isinstance(extra_ecs_path, list)) + self.assertEqual(len(extra_ecs_path), 2) + self.assertEqual(extra_ecs_path[0], os.path.join(self.test_prefix, 'files_pr123')) + self.assertEqual(extra_ecs_path[1], os.path.join(self.test_prefix, 'files_pr456')) + + review_pr_path = os.path.join(self.test_prefix, 'review_pr') + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123,456], + review_pr=789, from_commit='c0ff33') + self.assertEqual(tweaked_ecs_path, None) + self.assertTrue(extra_ecs_path) + self.assertTrue(isinstance(extra_ecs_path, list)) + self.assertEqual(len(extra_ecs_path), 4) + self.assertEqual(extra_ecs_path[0], os.path.join(self.test_prefix, 'files_pr123')) + self.assertEqual(extra_ecs_path[1], os.path.join(self.test_prefix, 'files_pr456')) + self.assertEqual(extra_ecs_path[2], os.path.join(self.test_prefix, 'files_pr789')) + self.assertEqual(extra_ecs_path[3], os.path.join(self.test_prefix, 'files_commit_c0ff33')) + def test_tweak_multiple_tcs(self): """Test that tweaking variables of ECs from multiple toolchains works""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') diff --git a/test/framework/robot.py b/test/framework/robot.py index 4f226bae35..4a23a21500 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -713,6 +713,46 @@ def test_search_paths(self): regex = re.compile(r"^ \* %s$" % os.path.join(self.test_prefix, test_ec), re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + def test_github_det_easyconfig_paths_from_commit(self): + """Test det_easyconfig_paths function in combination with --from-commit.""" + + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + + commit = '589282cf52609067616fc2a522f8e4b81f809cb7' + args = [ + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), # absolute path + 'toy-0.0-iter.eb', # relative path, available via robot search path + # commit in which ReFrame-4.3.2.eb was added, see https://github.com/easybuilders/easybuild-easyconfigs/pull/18763/commits + '--from-commit', commit, + 'ReFrame-4.3.2.eb', # easyconfig included in commit, should be resolved via robot search path + '--dry-run', + '--robot', + '--robot=%s' % test_ecs_path, + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % self.test_prefix, + ] + + self.mock_stderr(True) + outtxt = self.eb_main(args, raise_error=True) + stderr = self.get_stderr() + self.mock_stderr(False) + + self.assertFalse(stderr) + + # full path doesn't matter (helps to avoid failing tests due to resolved symlinks) + test_ecs_path = os.path.join('.*', 'test', 'framework', 'easyconfigs', 'test_ecs') + + modules = [ + (test_ecs_path, 'toy/0.0'), + (test_ecs_path, 'toy/0.0-iter'), + (os.path.join(self.test_prefix, '.*', 'files_commit_%s' % commit), 'ReFrame/4.3.2'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + def test_github_det_easyconfig_paths_from_pr(self): """Test det_easyconfig_paths function, with --from-pr enabled as well.""" if self.github_token is None: @@ -1082,8 +1122,8 @@ def test_tweak_robotpath(self): test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # Create directories to store the tweaked easyconfigs - tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) - robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_paths, auto_robot=True) + tweaked_ecs_paths, extra_ec_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) + robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, extra_ec_paths, auto_robot=True) init_config(build_options={ 'valid_module_classes': module_classes(), From 4363f9f371cbb94623e0be62c5ca3de37469fd88 Mon Sep 17 00:00:00 2001 From: Rovanion Luckey Date: Fri, 1 Mar 2024 10:40:09 +0100 Subject: [PATCH 128/430] filetools: Fix missing raw string for \; in get_source_tarball_from_git. --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 470e347e88..330c89366b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2680,7 +2680,7 @@ def get_source_tarball_from_git(filename, target_dir, git_config): # print names of all files and folders excluding .git directory 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', # reset access and modification timestamps - '-exec', 'touch', '-t 197001010100', '{}', '\;', '|', + '-exec', 'touch', '-t 197001010100', '{}', r'\;', '|', # sort file list 'LC_ALL=C', 'sort', '--zero-terminated', '|', # create tarball in GNU format with ownership reset From 07d06af44eeb3cbc70879a25b6dc28f4433cf195 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 1 Mar 2024 17:23:14 +0100 Subject: [PATCH 129/430] fix test_github_merge_pr by using more recent easyconfigs PR --- test/framework/options.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 26daa0cc7a..a491936d78 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4806,26 +4806,29 @@ def test_github_merge_pr(self): self.assertEqual(stderr.strip(), expected_stderr) self.assertTrue(stdout.strip().endswith(expected_stdout), "'%s' ends with '%s'" % (stdout, expected_stdout)) - # full eligible merged PR, default target branch + # full eligible merged PR, default target branch; + # note: we frequently need to change to a more recent PR here, + # to avoid that this test starts failing because commit status is set to None for old commits del args[-1] - args[1] = '17065' + # easyconfig PR for EasyBuild v4.8.2 + args[1] = '19105' stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) expected_stdout = '\n'.join([ - "Checking eligibility of easybuilders/easybuild-easyconfigs PR #17065 for merging...", + "Checking eligibility of easybuilders/easybuild-easyconfigs PR #19105 for merging...", "* targets develop branch: OK", "* test suite passes: OK", "* last test report is successful: OK", "* no pending change requests: OK", "* approved review: OK (by SebastianAchilles)", - "* milestone is set: OK (4.7.1)", + "* milestone is set: OK (4.9.0)", "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", '', - "[DRY RUN] Adding comment to easybuild-easyconfigs issue #17065: 'Going in, thanks @boegel!'", - "[DRY RUN] Merged easybuilders/easybuild-easyconfigs pull request #17065", + "[DRY RUN] Adding comment to easybuild-easyconfigs issue #19105: 'Going in, thanks @boegel!'", + "[DRY RUN] Merged easybuilders/easybuild-easyconfigs pull request #19105", ]) expected_stderr = '' self.assertEqual(stderr.strip(), expected_stderr) From ff4ff4934bd9a21be971d71a95685c67394ae72f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 1 Mar 2024 16:58:14 +0100 Subject: [PATCH 130/430] update easyblocks for toy extensions to make sure that asynchronous installation command is run in correct working directory --- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 15fe5773aa..28792ff4df 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -93,7 +93,7 @@ def run_async(self, thread_pool): cmd = f"echo 'no sources for {self.name}'" return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), - fail_on_error=False, task_id=task_id) + fail_on_error=False, task_id=task_id, work_dir=os.getcwd()) def postrun(self): """ diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index fb842bfe06..19b19d32dd 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -170,7 +170,7 @@ def run_async(self, thread_pool): cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) task_id = f'ext_{self.name}_{self.version}' return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), - fail_on_error=False, task_id=task_id) + fail_on_error=False, task_id=task_id, work_dir=os.getcwd()) def postrun(self): """ From cb6905601739d6f3437a29777a861eefc90ac96c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 1 Mar 2024 17:43:32 +0100 Subject: [PATCH 131/430] add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration --- .github/workflows/container_tests_apptainer.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 35c26c26c9..137b297379 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -29,10 +29,18 @@ jobs: - name: install OS & Python packages run: | # for building CentOS 7 container images - sudo apt-get install rpm - sudo apt-get install dnf + APT_PKGS="rpm dnf" # for modules tool - sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + + # Avoid apt-get update, as we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + if ! sudo apt-get install $APT_PKGS; then + # Try to update cache, then try again to resolve 404s of old packages + sudo apt-get update -yqq || true + sudo apt-get install $APT_PKGS + fi + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then From 2266b7166ccffd5826396955ca2b485657cca663 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 1 Mar 2024 17:04:16 +0000 Subject: [PATCH 132/430] append to `APT_PKGS` instead of overwriting it --- .github/workflows/container_tests_apptainer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 137b297379..b7fc2a8418 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -31,7 +31,7 @@ jobs: # for building CentOS 7 container images APT_PKGS="rpm dnf" # for modules tool - APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + APT_PKGS+="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" # Avoid apt-get update, as we don't really need it, # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) From 05ab883213870c7cf86d97d2be2fa05792063aba Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 1 Mar 2024 17:06:45 +0000 Subject: [PATCH 133/430] missing space --- .github/workflows/container_tests_apptainer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index b7fc2a8418..77d2a4a395 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -31,7 +31,7 @@ jobs: # for building CentOS 7 container images APT_PKGS="rpm dnf" # for modules tool - APT_PKGS+="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" # Avoid apt-get update, as we don't really need it, # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) From e3308507256a045b691c5d674a221804502f6d9a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 2 Mar 2024 10:44:33 +0100 Subject: [PATCH 134/430] add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration --- .github/workflows/container_tests_apptainer.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 1f46f2f5f6..1539221489 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -29,10 +29,18 @@ jobs: - name: install OS & Python packages run: | # for building CentOS 7 container images - sudo apt-get install rpm - sudo apt-get install dnf + APT_PKGS="rpm dnf" # for modules tool - sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + + # Avoid apt-get update, as we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + if ! sudo apt-get install $APT_PKGS; then + # Try to update cache, then try again to resolve 404s of old packages + sudo apt-get update -yqq || true + sudo apt-get install $APT_PKGS + fi + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then From 5377e40175229206de64bf71904b688c65dbcbfe Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 2 Mar 2024 13:55:15 +0000 Subject: [PATCH 135/430] replace `log.warn` with `log.warning` --- easybuild/tools/filetools.py | 2 +- test/framework/build_log.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1208be1d07..d0959723f8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2240,7 +2240,7 @@ def det_size(path): if os.path.exists(fullpath): installsize += os.path.getsize(fullpath) except OSError as err: - _log.warn("Could not determine install size: %s" % err) + _log.warning("Could not determine install size: %s" % err) return installsize diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 21755b62b3..dcaa1620c3 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -208,7 +208,7 @@ def test_log_levels(self): log.error('kaput') log.deprecated('almost kaput', '10000000000000') log.raiseError = True - log.warn('this is a warning') + log.warning('this is a warning') log.info('fyi') log.debug('gdb') log.devel('tmi') From 57839104045f839476536330a0cb53da8d586bc4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Mar 2024 12:03:03 +0100 Subject: [PATCH 136/430] add support to run_shell_cmd for providing list of answers in qa_patterns + fix output for hooks triggered for interactive shell commands run with run_shell_cmd --- easybuild/tools/run.py | 20 ++++++++++++++++++-- test/framework/run.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c9a4b3d06f..ba40b704ac 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -319,7 +319,11 @@ def to_cmd_str(cmd): if with_hooks: hooks = load_hooks(build_option('hooks')) - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': work_dir}) + kwargs = { + 'interactive': bool(qa_patterns), + 'work_dir': work_dir, + } + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=kwargs) if hook_res: cmd, old_cmd = hook_res, cmd cmd_str = to_cmd_str(cmd) @@ -375,10 +379,21 @@ def to_cmd_str(cmd): # only consider answering questions if there's new output beyond additional whitespace if qa_patterns: - for question, answer in qa_patterns: + for question, answers in qa_patterns: + question += r'[\s\n]*$' regex = re.compile(question.encode()) if regex.search(stdout): + # if answer is specified as a list, we take the first item as current answer, + # and add it to the back of the list (so we cycle through answers) + if isinstance(answers, list): + answer = answers.pop(0) + answers.append(answer) + elif isinstance(answers, str): + answer = answers + else: + raise EasyBuildError(f"Unknown type of answers encountered: {answers}") + answer += '\n' os.write(proc.stdin.fileno(), answer.encode()) time_no_match = 0 @@ -437,6 +452,7 @@ def to_cmd_str(cmd): if with_hooks: run_hook_kwargs = { 'exit_code': res.exit_code, + 'interactive': bool(qa_patterns), 'output': res.output, 'stderr': res.stderr, 'work_dir': res.work_dir, diff --git a/test/framework/run.py b/test/framework/run.py index aa22ef88fd..6fb8d2b072 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -996,6 +996,27 @@ def test_run_cmd_qa_answers(self): self.assertEqual(out, "question\nanswer1\nquestion\nanswer2\n" * 2) self.assertEqual(ec, 0) + def test_run_shell_cmd_qa_answers(self): + """Test providing list of answers for a question in run_shell_cmd.""" + + cmd = "echo question; read x; echo $x; " * 2 + qa = [("question", ["answer1", "answer2"])] + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.output, "question\nanswer1\nquestion\nanswer2\n") + self.assertEqual(res.exit_code, 0) + + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, "Unknown type of answers encountered", run_shell_cmd, cmd, qa_patterns=[('question', 1)]) + + # test cycling of answers + cmd = cmd * 2 + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.output, "question\nanswer1\nquestion\nanswer2\n" * 2) + self.assertEqual(res.exit_code, 0) + def test_run_cmd_simple(self): """Test return value for run_cmd in 'simple' mode.""" with self.mocked_stdout_stderr(): @@ -1124,6 +1145,14 @@ def test_run_cmd_dry_run(self): self.assertEqual(read_file(outfile), "This is always echoed\n") # Q&A commands + self.mock_stdout(True) + run_shell_cmd("some_qa_cmd", qa_patterns=[('question1', 'answer1')]) + stdout = self.get_stdout() + self.mock_stdout(False) + + expected = """ running interactive shell command "some_qa_cmd"\n""" + self.assertIn(expected, stdout) + self.mock_stdout(True) run_cmd_qa("some_qa_cmd", {'question1': 'answer1'}) stdout = self.get_stdout() @@ -1565,6 +1594,17 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ]) self.assertEqual(stdout, expected_stdout) + with self.mocked_stdout_stderr(): + run_shell_cmd("sleep 2; make", qa_patterns=[('q', 'a')]) + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook interactive 'sleep 2; make' in %s" % cwd, + "post-run hook interactive 'sleep 2; echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + with self.mocked_stdout_stderr(): run_cmd_qa("sleep 2; make", qa={}) stdout = self.get_stdout() From e9cec346425783205431ebe296d186d4df3aad6e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 14:58:16 -0500 Subject: [PATCH 137/430] Eliminate use of distutils.version.StrictVersion distutils was removed in Python 3.12. The only reason EasyBuild uses StrictVersion is that it orders beta/rc versions before the released version, unlike LooseVersion. E.g. 5.0.0-beta < 5.0.0 (but > for LooseVersion). So a new method `is_earlier_or_prerelease(self, other, markers)` was added to LooseVersion to handle that particular case. Addresses part of #3963 --- easybuild/tools/__init__.py | 9 --------- easybuild/tools/loose_version.py | 16 ++++++++++++++++ easybuild/tools/modules.py | 27 ++++++++++----------------- test/framework/modules.py | 14 ++++++++------ test/framework/modulestool.py | 6 +++--- test/framework/utilities_test.py | 4 ++++ 6 files changed, 41 insertions(+), 35 deletions(-) diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index dee4fc0d12..bd36b15edb 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -37,14 +37,5 @@ __path__ = __import__('pkgutil').extend_path(__path__, __name__) -import distutils.version import warnings from easybuild.tools.loose_version import LooseVersion # noqa(F401) - - -class StrictVersion(distutils.version.StrictVersion): - """Temporary wrapper over distuitls StrictVersion that silences the deprecation warning""" - def __init__(self, *args, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - distutils.version.StrictVersion.__init__(self, *args, **kwargs) diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py index 1855fee74a..db374b20ca 100644 --- a/easybuild/tools/loose_version.py +++ b/easybuild/tools/loose_version.py @@ -53,6 +53,22 @@ def version(self): """Readonly access to the parsed version (list or None)""" return self._version + def is_earlier_or_prerelease(self, other, markers): + """Check if this is an earlier version or prerelease of other + + Markers is a list of strings that denote a prerelease + """ + if isinstance(other, str): + vstring = other + else: + vstring = other._vstring + if self._vstring.startswith(vstring): + prerelease = self._vstring[len(vstring):] + for marker in markers: + if prerelease.startswith(marker): + return True + return self < other + def __str__(self): return self._vstring diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 75a1022b50..c2bbf71065 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -42,7 +42,7 @@ import shlex from easybuild.base import fancylogger -from easybuild.tools import StrictVersion +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS @@ -144,11 +144,11 @@ class ModulesTool(object): COMMAND_SHELL = None # option to determine the version VERSION_OPTION = '--version' - # minimal required version (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion)) + # minimal required version (cannot include -beta or rc) REQ_VERSION = None # deprecated version limit (support for versions below this version is deprecated) DEPR_VERSION = None - # maximum version allowed (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion)) + # maximum version allowed (cannot include -beta or rc) MAX_VERSION = None # the regexp, should have a "version" group (multiline search) VERSION_REGEXP = None @@ -239,14 +239,6 @@ def set_and_check_version(self): if res: self.version = res.group('version') self.log.info("Found %s version %s", self.NAME, self.version) - - # make sure version is a valid StrictVersion (e.g., 5.7.3.1 is invalid), - # and replace 'rc' by 'b', to make StrictVersion treat it as a beta-release - self.version = self.version.replace('rc', 'b').replace('-beta', 'b1') - if len(self.version.split('.')) > 3: - self.version = '.'.join(self.version.split('.')[:3]) - - self.log.info("Converted actual version to '%s'" % self.version) else: raise EasyBuildError("Failed to determine %s version from option '%s' output: %s", self.NAME, self.VERSION_OPTION, txt) @@ -259,9 +251,10 @@ def set_and_check_version(self): elif build_option('modules_tool_version_check'): self.log.debug("Checking whether %s version %s meets requirements", self.NAME, self.version) + version = LooseVersion(self.version) if self.REQ_VERSION is not None: self.log.debug("Required minimum %s version defined: %s", self.NAME, self.REQ_VERSION) - if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): + if version.is_earlier_or_prerelease(self.REQ_VERSION, ['rc', '-beta']): raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s", self.NAME, self.REQ_VERSION, self.version) else: @@ -269,14 +262,14 @@ def set_and_check_version(self): if self.DEPR_VERSION is not None: self.log.debug("Deprecated %s version limit defined: %s", self.NAME, self.DEPR_VERSION) - if StrictVersion(self.version) < StrictVersion(self.DEPR_VERSION): + if version.is_earlier_or_prerelease(self.DEPR_VERSION, ['rc', '-beta']): depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) - if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION): + if version.is_earlier_or_prerelease(self.MAX_VERSION, ['rc', '-beta']): raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s", self.NAME, self.MAX_VERSION, self.version) else: @@ -1390,7 +1383,7 @@ def available(self, mod_name=None, extra_args=None): if extra_args is None: extra_args = [] # make hidden modules visible (requires Environment Modules 4.6.0) - if StrictVersion(self.version) >= StrictVersion('4.6.0'): + if LooseVersion(self.version) >= LooseVersion('4.6.0'): extra_args.append(self.SHOW_HIDDEN_OPTION) return super(EnvironmentModules, self).available(mod_name=mod_name, extra_args=extra_args) @@ -1440,11 +1433,11 @@ def __init__(self, *args, **kwargs): setvar('LMOD_EXTENDED_DEFAULT', 'no', verbose=False) super(Lmod, self).__init__(*args, **kwargs) - version = StrictVersion(self.version) + version = LooseVersion(self.version) self.supports_depends_on = True # See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html - if version >= '8.7.12': + if version >= LooseVersion('8.7.12'): self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod') else: self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') diff --git a/test/framework/modules.py b/test/framework/modules.py index a849148bdf..3e9c7343ba 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -42,7 +42,7 @@ import easybuild.tools.modules as mod from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.easyconfig import EasyConfig -from easybuild.tools import StrictVersion +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.environment import modify_env from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir @@ -226,15 +226,16 @@ def test_avail(self): # all test modules are accounted for ms = self.modtool.available() + version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('5.7.5'): + if isinstance(self.modtool, Lmod) and not version.is_prerelease_or_earlier('5.7.5', ['rc']): # with recent versions of Lmod, also the hidden modules are included in the output of 'avail' self.assertEqual(len(ms), TEST_MODULES_COUNT + 3) self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) - and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')): + and not version.is_prerelease_or_earlier('4.6.0', ['-beta'])): # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) @@ -314,7 +315,8 @@ def test_exist(self): avail_mods = self.modtool.available() self.assertIn('Java/1.8.0_181', avail_mods) - if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): + version = LooseVersion(self.modtool.version) + if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.0', ['rc']): self.assertIn('Java/1.8', avail_mods) self.assertIn('Java/site_default', avail_mods) self.assertIn('JavaAlias', avail_mods) @@ -374,7 +376,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [True, True]) # also check with .modulerc.lua for Lmod 7.8 or newer - if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.8'): + if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.8', ['rc']): shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir) reset_module_caches() @@ -406,7 +408,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) # Test alias in home directory .modulerc - if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): + if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.0', ['rc']): # Required or temporary HOME would be in MODULEPATH already self.init_testmods() # Sanity check: Module aliases don't exist yet diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index f43a91e3b3..3c3eae74f5 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -36,7 +36,7 @@ from unittest import TextTestRunner from easybuild.base import fancylogger -from easybuild.tools import modules, StrictVersion +from easybuild.tools import modules, LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, which, write_file from easybuild.tools.modules import EnvironmentModules, Lmod @@ -76,7 +76,7 @@ def test_mock(self): mmt = MockModulesTool(mod_paths=[], testing=True) # the version of the MMT is the commandline option - self.assertEqual(mmt.version, StrictVersion(MockModulesTool.VERSION_OPTION)) + self.assertEqual(mmt.version, LooseVersion(MockModulesTool.VERSION_OPTION)) cmd_abspath = which(MockModulesTool.COMMAND) @@ -100,7 +100,7 @@ def test_environment_command(self): bmmt = BrokenMockModulesTool(mod_paths=[], testing=True) cmd_abspath = which(MockModulesTool.COMMAND) - self.assertEqual(bmmt.version, StrictVersion(MockModulesTool.VERSION_OPTION)) + self.assertEqual(bmmt.version, LooseVersion(MockModulesTool.VERSION_OPTION)) self.assertEqual(bmmt.cmd, cmd_abspath) # clean it up diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index ba4766f302..6e9421681f 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -143,6 +143,10 @@ def test_LooseVersion(self): # Careful here: 1.0 > 1 !!! self.assertGreater(LooseVersion('1.0'), LooseVersion('1')) self.assertLess(LooseVersion('1'), LooseVersion('1.0')) + # checking prereleases + self.assertGreater(LooseVersion('4.0.0-beta'), LooseVersion('4.0.0')) + self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0'), ['-beta'], True) + self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0'), ['rc'], False) # The following test is taken from Python distutils tests # licensed under the Python Software Foundation License Version 2 From 7d9762088be7c1db05166aff1336e6b0ecb22346 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 15:07:02 -0500 Subject: [PATCH 138/430] Remove unused warnings imported, spotted by flake8. --- easybuild/tools/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index bd36b15edb..cc7eadb99f 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -37,5 +37,4 @@ __path__ = __import__('pkgutil').extend_path(__path__, __name__) -import warnings from easybuild.tools.loose_version import LooseVersion # noqa(F401) From 5c6b7f7f1a60f548b4fd7a69a6360695c93e21cb Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 15:27:09 -0500 Subject: [PATCH 139/430] Eliminate `distutils.util.strtobool`. It's only used once so I open-coded it. A stricter but not backwards compatible check would be to only allow 'True' and 'False'. --- easybuild/framework/easyconfig/types.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index fa9829e312..2cf9b77058 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -31,7 +31,6 @@ * Caroline De Brouwer (Ghent University) * Kenneth Hoste (Ghent University) """ -from distutils.util import strtobool from easybuild.base import fancylogger from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS @@ -280,7 +279,14 @@ def to_toolchain_dict(spec): res = {'name': spec[0].strip(), 'version': spec[1].strip()} # 3-element list elif len(spec) == 3: - res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': strtobool(spec[2].strip())} + hidden = spec[2].strip().lower() + if hidden in {'yes', 'true', 't', 'y', '1'}: + hidden = True + elif hidden in {'no', 'false', 'f', 'n', '0'}: + hidden = False + else: + raise EasyBuildError("Invalid truth value %s", hidden) + res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': hidden} else: raise EasyBuildError("Can not convert list %s to toolchain dict. Expected 2 or 3 elements", spec) From 03c0b3d238dfa1e804abc58d2270e79311cfafd6 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 15:54:47 -0500 Subject: [PATCH 140/430] Fix spelling of method call. --- test/framework/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 3e9c7343ba..c51312698a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -228,14 +228,14 @@ def test_avail(self): ms = self.modtool.available() version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_prerelease_or_earlier('5.7.5', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('5.7.5', ['rc']): # with recent versions of Lmod, also the hidden modules are included in the output of 'avail' self.assertEqual(len(ms), TEST_MODULES_COUNT + 3) self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) - and not version.is_prerelease_or_earlier('4.6.0', ['-beta'])): + and not version.is_earlier_or_prerelease('4.6.0', ['-beta'])): # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) From 303f453b1b8ddb688e34ff6fede3cb9e57dcb64b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 15:56:33 -0500 Subject: [PATCH 141/430] Fix brackets for is_earlier_or_prerelease call --- test/framework/utilities_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 6e9421681f..6e7487bae8 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -145,8 +145,8 @@ def test_LooseVersion(self): self.assertLess(LooseVersion('1'), LooseVersion('1.0')) # checking prereleases self.assertGreater(LooseVersion('4.0.0-beta'), LooseVersion('4.0.0')) - self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0'), ['-beta'], True) - self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0'), ['rc'], False) + self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0', ['-beta']), True) + self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0', ['rc']), False) # The following test is taken from Python distutils tests # licensed under the Python Software Foundation License Version 2 From 9cd91f0ff5fd9321545c987484985d93ea08bfba Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 16:00:00 -0500 Subject: [PATCH 142/430] Check for EasyBuildError for strtobool replacement --- test/framework/type_checking.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index 1eb2f19fd3..e52c3da603 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -317,9 +317,9 @@ def test_to_toolchain_dict(self): self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, ['gcc', '4', 'False', '7']) # invalid truth value - errstr = "invalid truth value .*" - self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, "intel, 2015, foo") - self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, ['gcc', '4', '7']) + errstr = "Invalid truth value .*" + self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, "intel, 2015, foo") + self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, ['gcc', '4', '7']) # missing keys self.assertErrorRegex(EasyBuildError, "Incorrect set of keys", to_toolchain_dict, {'name': 'intel'}) From 8f2265131883e0eb6b75c25b640228e0e097314b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 17:02:40 -0500 Subject: [PATCH 143/430] Use is_prerelase instead of is_earlier_or_prerelease. In the end this is clearer because we also need to test against a maximum version (but it is allowed to be a prerelease of that maximum version, as per StrictVersion semantics!) --- easybuild/tools/loose_version.py | 6 +++--- easybuild/tools/modules.py | 6 +++--- test/framework/modules.py | 10 +++++----- test/framework/utilities_test.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py index db374b20ca..fb2e0fcb78 100644 --- a/easybuild/tools/loose_version.py +++ b/easybuild/tools/loose_version.py @@ -53,8 +53,8 @@ def version(self): """Readonly access to the parsed version (list or None)""" return self._version - def is_earlier_or_prerelease(self, other, markers): - """Check if this is an earlier version or prerelease of other + def is_prerelease(self, other, markers): + """Check if this is a prerelease of other Markers is a list of strings that denote a prerelease """ @@ -67,7 +67,7 @@ def is_earlier_or_prerelease(self, other, markers): for marker in markers: if prerelease.startswith(marker): return True - return self < other + return False def __str__(self): return self._vstring diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index c2bbf71065..ae358e3bbe 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -254,7 +254,7 @@ def set_and_check_version(self): version = LooseVersion(self.version) if self.REQ_VERSION is not None: self.log.debug("Required minimum %s version defined: %s", self.NAME, self.REQ_VERSION) - if version.is_earlier_or_prerelease(self.REQ_VERSION, ['rc', '-beta']): + if version.is_prerelease(self.REQ_VERSION, ['rc', '-beta']) or version < self.REQ_VERSION: raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s", self.NAME, self.REQ_VERSION, self.version) else: @@ -262,14 +262,14 @@ def set_and_check_version(self): if self.DEPR_VERSION is not None: self.log.debug("Deprecated %s version limit defined: %s", self.NAME, self.DEPR_VERSION) - if version.is_earlier_or_prerelease(self.DEPR_VERSION, ['rc', '-beta']): + if version.is_prerelease(self.DEPR_VERSION, ['rc', '-beta']) or version < self.DEPR_VERSION: depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) - if version.is_earlier_or_prerelease(self.MAX_VERSION, ['rc', '-beta']): + if not version.is_prerelease(self.MAX_VERSION, ['rc', '-beta']) and self.version > self.MAX_VERSION: raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s", self.NAME, self.MAX_VERSION, self.version) else: diff --git a/test/framework/modules.py b/test/framework/modules.py index c51312698a..efcad01d5a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -228,14 +228,14 @@ def test_avail(self): ms = self.modtool.available() version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('5.7.5', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_prerelease('5.7.5', ['rc']) and version >= '5.7.5': # with recent versions of Lmod, also the hidden modules are included in the output of 'avail' self.assertEqual(len(ms), TEST_MODULES_COUNT + 3) self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) - and not version.is_earlier_or_prerelease('4.6.0', ['-beta'])): + and not version.is_prerelease('4.6.0', ['-beta'])) and version >= '4.6.0': # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) @@ -316,7 +316,7 @@ def test_exist(self): avail_mods = self.modtool.available() self.assertIn('Java/1.8.0_181', avail_mods) version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.0', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.0', ['rc']) and version >= '7.0': self.assertIn('Java/1.8', avail_mods) self.assertIn('Java/site_default', avail_mods) self.assertIn('JavaAlias', avail_mods) @@ -376,7 +376,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [True, True]) # also check with .modulerc.lua for Lmod 7.8 or newer - if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.8', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.8', ['rc']) and version >= '7.8': shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir) reset_module_caches() @@ -408,7 +408,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) # Test alias in home directory .modulerc - if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.0', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.0', ['rc']) or version >= '7.0': # Required or temporary HOME would be in MODULEPATH already self.init_testmods() # Sanity check: Module aliases don't exist yet diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 6e7487bae8..2e862e6ad6 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -145,8 +145,8 @@ def test_LooseVersion(self): self.assertLess(LooseVersion('1'), LooseVersion('1.0')) # checking prereleases self.assertGreater(LooseVersion('4.0.0-beta'), LooseVersion('4.0.0')) - self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0', ['-beta']), True) - self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0', ['rc']), False) + self.assertEqual(LooseVersion('4.0.0-beta').is_prerelease('4.0.0', ['-beta']), True) + self.assertEqual(LooseVersion('4.0.0-beta').is_prerelease('4.0.0', ['rc']), False) # The following test is taken from Python distutils tests # licensed under the Python Software Foundation License Version 2 From bce95ecea3f5b5bcbc244dfb2c39a3227bd5e2d8 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 7 Mar 2024 09:56:02 -0500 Subject: [PATCH 144/430] Apply suggestions from code review Allow `on` and `off` as well for compatibility with `distutils.strtobool` Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/framework/easyconfig/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index 2cf9b77058..451d22af03 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -280,9 +280,9 @@ def to_toolchain_dict(spec): # 3-element list elif len(spec) == 3: hidden = spec[2].strip().lower() - if hidden in {'yes', 'true', 't', 'y', '1'}: + if hidden in {'yes', 'true', 't', 'y', '1', 'on'}: hidden = True - elif hidden in {'no', 'false', 'f', 'n', '0'}: + elif hidden in {'no', 'false', 'f', 'n', '0', 'off'}: hidden = False else: raise EasyBuildError("Invalid truth value %s", hidden) From afc36be43c0e1f8fe2bd8a44f948f4b44eab4472 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 7 Mar 2024 19:48:32 +0000 Subject: [PATCH 145/430] ensure parameters in DEPRECATED_PARAMETERS are not treated as unknown --- easybuild/framework/easyconfig/easyconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 6e0d2aa141..7f123be9dc 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -179,7 +179,7 @@ def triage_easyconfig_params(variables, ec): for key in variables: # validations are skipped, just set in the config - if key in ec: + if key in ec or key in DEPRECATED_PARAMETERS.keys(): ec_params[key] = variables[key] _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key])) elif key in REPLACED_PARAMETERS: @@ -658,7 +658,7 @@ def set_keys(self, params): with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config - if key in self._config.keys(): + if key in self._config.keys() or key in DEPRECATED_PARAMETERS.keys(): self[key] = params[key] self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) From b8290964c848d8868e792bfe33d3fc45242d09ae Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 7 Mar 2024 20:07:29 +0000 Subject: [PATCH 146/430] update deprecated functionality docs url --- test/framework/build_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/build_log.py b/test/framework/build_log.py index dcaa1620c3..581a995c24 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -116,7 +116,7 @@ def test_easybuildlog(self): stderr = self.get_stderr() self.mock_stderr(False) - more_info = "see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html for more information" + more_info = "see https://docs.easybuild.io/deprecated-functionality/ for more information" expected_stderr = '\n\n'.join([ "\nWARNING: Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info, "\nWARNING: Deprecated functionality, will no longer work in v2.0: onemorewarning", From d2b3c7d17c0533814056bcb458d12666c1a13e10 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 7 Mar 2024 20:09:25 +0000 Subject: [PATCH 147/430] remove period at end of easyconfig parameter deprecation warning --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 6e0d2aa141..f03f2549e7 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -121,7 +121,7 @@ def new_ec_method(self, key, *args, **kwargs): if key in DEPRECATED_PARAMETERS: depr_key = key key, ver = DEPRECATED_PARAMETERS[depr_key] - _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead." % (depr_key, key), ver) + _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver) if key in REPLACED_PARAMETERS: _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') return ec_method(self, key, *args, **kwargs) From 4b873ecff163612814ef032ab753e1cd14ca7f99 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 7 Mar 2024 20:15:34 +0000 Subject: [PATCH 148/430] update easybuild.readthedocs.io urls to use docs.easybuild.io --- easybuild/framework/easyconfig/easyconfig.py | 4 ++-- easybuild/tools/build_log.py | 2 +- easybuild/tools/github.py | 2 +- easybuild/tools/modules.py | 2 +- easybuild/tools/toolchain/compiler.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index f03f2549e7..b7763d02d6 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -792,7 +792,7 @@ def local_var_naming(self, local_var_naming_check): msg = "Use of %d unknown easyconfig parameters detected %s: %s\n" % (cnt, in_fn, unknown_keys_msg) msg += "If these are just local variables please rename them to start with '%s', " % LOCAL_VAR_PREFIX msg += "or try using --fix-deprecated-easyconfigs to do this automatically.\nFor more information, see " - msg += "https://easybuild.readthedocs.io/en/latest/Easyconfig-files-local-variables.html ." + msg += "https://docs.easybuild.io/easyconfig-files-local-variables/ ." # always log a warning if local variable that don't follow recommended naming scheme are found self.log.warning(msg) @@ -830,7 +830,7 @@ def check_deprecated(self, path): depr_maj_ver = int(str(VERSION).split('.')[0]) + 1 depr_ver = '%s.0' % depr_maj_ver - more_info_depr_ec = " (see also http://easybuild.readthedocs.org/en/latest/Deprecated-easyconfigs.html)" + more_info_depr_ec = " (https://docs.easybuild.io/deprecated-easyconfigs)" self.log.deprecated(depr_msg, depr_ver, more_info=more_info_depr_ec, silent=build_option('silent')) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index f8529fff57..04bcaae16c 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -55,7 +55,7 @@ # allow some experimental experimental code EXPERIMENTAL = False -DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html' +DEPRECATED_DOC_URL = 'https://docs.easybuild.io/deprecated-functionality/' DRY_RUN_BUILD_DIR = None DRY_RUN_SOFTWARE_INSTALL_DIR = None diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 875724acb4..f366985b7f 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2178,7 +2178,7 @@ def check_github(): msg = '\n'.join([ '', "One or more checks FAILed, GitHub configuration not fully complete!", - "See http://easybuild.readthedocs.org/en/latest/Integration_with_GitHub.html#configuration for help.", + "See https://docs.easybuild.io/integration-with-github/#github_configuration for help.", '', ]) print_msg(msg, log=_log, prefix=False) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 75a1022b50..8c51ec57b5 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -945,7 +945,7 @@ def check_loaded_modules(self): "use the --allow-loaded-modules configuration option.", "To specify action to take when loaded modules are detected, use %s." % opt, '', - "See http://easybuild.readthedocs.io/en/latest/Detecting_loaded_modules.html for more information.", + "See https://docs.easybuild.io/detecting-loaded-modules/ for more information.", ]) action = build_option('detect_loaded_modules') diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 6387273571..b70ebed589 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -358,9 +358,9 @@ def _set_optimal_architecture(self, default_optarch=None): optarch_flags_str = "%soptarch flags" % ('', 'generic ')[use_generic] error_msg = "Don't know how to set %s for %s/%s! " % (optarch_flags_str, self.arch, self.cpu_family) error_msg += "Use --optarch='' to override (see " - error_msg += "http://easybuild.readthedocs.io/en/latest/Controlling_compiler_optimization_flags.html " + error_msg += "https://docs.easybuild.io/controlling-compiler-optimization-flags/ " error_msg += "for details) and consider contributing your settings back (see " - error_msg += "http://easybuild.readthedocs.io/en/latest/Contributing.html)." + error_msg += "https://docs.easybuild.io/contributing/)." raise EasyBuildError(error_msg) def comp_family(self, prefix=None): From ec7be23acd1c7c6e72e0d2681024c5ca7a11adda Mon Sep 17 00:00:00 2001 From: Jasper Grimm <65227842+jfgrimm@users.noreply.github.com> Date: Thu, 7 Mar 2024 20:46:16 +0000 Subject: [PATCH 149/430] reinstate "see also" accidentally removed --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b7763d02d6..aab3674cd3 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -830,7 +830,7 @@ def check_deprecated(self, path): depr_maj_ver = int(str(VERSION).split('.')[0]) + 1 depr_ver = '%s.0' % depr_maj_ver - more_info_depr_ec = " (https://docs.easybuild.io/deprecated-easyconfigs)" + more_info_depr_ec = " (see also https://docs.easybuild.io/deprecated-easyconfigs)" self.log.deprecated(depr_msg, depr_ver, more_info=more_info_depr_ec, silent=build_option('silent')) From 63acc8f2112ed24d1b564067c82599c5d1c1c726 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 8 Mar 2024 10:31:22 +0100 Subject: [PATCH 150/430] Improve logging & handling of --optarch --- easybuild/tools/toolchain/compiler.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 32634032fc..5d9f025172 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -323,7 +323,7 @@ def _set_optimal_architecture(self, default_optarch=None): optarch = build_option('optarch') # --optarch is specified with flags to use - if optarch is not None and isinstance(optarch, dict): + if isinstance(optarch, dict): # optarch has been validated as complex string with multiple compilers and converted to a dictionary # first try module names, then the family in optarch current_compiler_names = (getattr(self, 'COMPILER_MODULE_NAME', []) + @@ -338,14 +338,12 @@ def _set_optimal_architecture(self, default_optarch=None): self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.", current_compiler) - use_generic = False - if optarch is not None: - # optarch has been parsed as a simple string - if isinstance(optarch, string_type): - if optarch == OPTARCH_GENERIC: - use_generic = True - else: - raise EasyBuildError("optarch is neither an string or a dict %s. This should never happen", optarch) + if isinstance(optarch, string_type): + use_generic = (optarch == OPTARCH_GENERIC) + elif optarch is None: + use_generic = False + else: + raise EasyBuildError("optarch is neither an string or a dict %s. This should never happen", optarch) if use_generic: if (self.arch, self.cpu_family) in (self.COMPILER_GENERIC_OPTION or []): @@ -360,10 +358,11 @@ def _set_optimal_architecture(self, default_optarch=None): optarch = self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(self.arch, self.cpu_family)] if optarch is not None: - self.log.info("_set_optimal_architecture: using %s as optarch for %s.", optarch, self.arch) + optarch_log_str = optarch or 'no flags' + self.log.info("_set_optimal_architecture: using %s as optarch for %s/%s.", + optarch_log_str, self.arch, self.cpu_family) self.options.options_map['optarch'] = optarch - - if self.options.options_map.get('optarch', None) is None: + elif self.options.options_map.get('optarch', None) is None: optarch_flags_str = "%soptarch flags" % ('', 'generic ')[use_generic] error_msg = "Don't know how to set %s for %s/%s! " % (optarch_flags_str, self.arch, self.cpu_family) error_msg += "Use --optarch='' to override (see " From 5c4fd2f12a8b0f769c5e134e7117d7d8f35ac2a1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 8 Mar 2024 18:19:20 +0100 Subject: [PATCH 151/430] enhance test_deprecated_easyconfig_parameters to cover bugs fixed when there are deprecated easyconfig parameters --- test/framework/easyconfig.py | 46 ++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 141e9b0f77..4a1e1df608 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1733,7 +1733,7 @@ def foo(key): self.assertErrorRegex(EasyBuildError, error_regex, foo, key) def test_deprecated_easyconfig_parameters(self): - """Test handling of replaced easyconfig parameters.""" + """Test handling of deprecated easyconfig parameters.""" os.environ.pop('EASYBUILD_DEPRECATED') easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version init_config() @@ -1744,10 +1744,13 @@ def test_deprecated_easyconfig_parameters(self): orig_deprecated_parameters = copy.deepcopy(easyconfig.parser.DEPRECATED_PARAMETERS) easyconfig.parser.DEPRECATED_PARAMETERS.update({ 'foobar': ('barfoo', '0.0'), # deprecated since forever - 'foobarbarfoo': ('barfoofoobar', '1000000000'), # won't be actually deprecated for a while + # won't be actually deprecated for a while; + # note that we should map foobarbarfoo to a valid easyconfig parameter here, + # or we'll hit errors when parsing an easyconfig file that uses it + 'foobarbarfoo': ('required_linked_shared_libs', '1000000000'), }) - # copy classes before reloading, so we can restore them (other isinstance checks fail) + # copy classes before reloading, so we can restore them (otherwise isinstance checks fail) orig_EasyConfig = copy.deepcopy(easyconfig.easyconfig.EasyConfig) orig_ActiveMNS = copy.deepcopy(easyconfig.easyconfig.ActiveMNS) reload(easyconfig.parser) @@ -1772,6 +1775,35 @@ def foo(key): self.assertEqual(ec[newkey], '123test') self.assertEqual(ec[key], '123test') + variables = { + 'name': 'example', + 'version': '1.2.3', + 'foobar': 'foobar', + 'local_var': 'test', + } + ec = { + 'name': None, + 'version': None, + 'homepage': None, + 'toolchain': None, + } + ec_params, unknown_keys = triage_easyconfig_params(variables, ec) + # deprecated easyconfig parameter 'foobar' is retained as easyconfig parameter; + # only local_var is not retained, since that's a local variable + self.assertEqual(unknown_keys, []) + expected = {'name': 'example', 'version': '1.2.3', 'foobar': 'foobar'} + self.assertEqual(ec_params, expected) + + # try parsing an easyconfig file that defines a deprecated easyconfig parameter + toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, read_file(toy_ec)) + write_file(test_ec, "\nfoobarbarfoo = 'foobarbarfoo'", append=True) + + with self.mocked_stdout_stderr(): + ec = EasyConfig(test_ec) + self.assertEqual(ec['required_linked_shared_libs'], 'foobarbarfoo') + easyconfig.parser.DEPRECATED_PARAMETERS = orig_deprecated_parameters reload(easyconfig.parser) reload(easyconfig.easyconfig) @@ -4288,15 +4320,19 @@ def test_triage_easyconfig_params(self): self.assertEqual(sorted(unknown_keys), ['bleh', 'foobar']) # check behaviour when easyconfig parameters that use a name indicating a local variable were defined - ec.update({ + local_vars = { 'x': None, 'local_foo': None, '_foo': None, '_': None, - }) + } + ec.update(local_vars) error = "Found 4 easyconfig parameters that are considered local variables: _, _foo, local_foo, x" self.assertErrorRegex(EasyBuildError, error, triage_easyconfig_params, variables, ec) + for key in local_vars: + del ec[key] + def test_local_vars_detection(self): """Test detection of using unknown easyconfig parameters that are likely local variables.""" From cc615db3136ddabe2882729363c7624337c7924c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 11 Mar 2024 09:59:50 +0100 Subject: [PATCH 152/430] fix code style issues --- easybuild/tools/github.py | 11 +++++++---- test/framework/easyconfig.py | 5 ++--- test/framework/robot.py | 3 ++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 0cfc181120..c8058d8a8c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -355,7 +355,8 @@ def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token= return res -def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, account=GITHUB_EB_MAIN, path=None, github_user=None): +def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, account=GITHUB_EB_MAIN, path=None, + github_user=None): """ Download entire GitHub repo as a tar.gz archive, and extract it into specified path. :param repo: repo to download @@ -475,7 +476,8 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi if len(cands) == 1: path = cands[0] else: - raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, extra_ec_paths) + raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", + pr, extra_ec_paths) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr) @@ -606,7 +608,8 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, if len(cands) == 1: path = cands[0] else: - raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s", commit, extra_ec_paths) + raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s", + commit, extra_ec_paths) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_commit_' + commit) @@ -640,7 +643,7 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, # if only filename is specified, we need to determine the file path if file == os.path.basename(file): src_path = None - for (dirpath, dirnames, filenames) in os.walk(repo_commit, topdown=True): + for (dirpath, _, filenames) in os.walk(repo_commit, topdown=True): if file in filenames: src_path = os.path.join(dirpath, file) break diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 2cadf46049..baa031c468 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -748,7 +748,7 @@ def test_alt_easyconfig_paths(self): self.assertEqual(tweaked_ecs_path[1], os.path.join(self.test_prefix, 'tweaked_dep_easyconfigs')) self.assertEqual(extra_ecs_path, []) - tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123,456]) + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123, 456]) self.assertEqual(tweaked_ecs_path, None) self.assertTrue(extra_ecs_path) self.assertTrue(isinstance(extra_ecs_path, list)) @@ -756,8 +756,7 @@ def test_alt_easyconfig_paths(self): self.assertEqual(extra_ecs_path[0], os.path.join(self.test_prefix, 'files_pr123')) self.assertEqual(extra_ecs_path[1], os.path.join(self.test_prefix, 'files_pr456')) - review_pr_path = os.path.join(self.test_prefix, 'review_pr') - tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123,456], + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123, 456], review_pr=789, from_commit='c0ff33') self.assertEqual(tweaked_ecs_path, None) self.assertTrue(extra_ecs_path) diff --git a/test/framework/robot.py b/test/framework/robot.py index 4a23a21500..ccdf9a2b1e 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -722,7 +722,8 @@ def test_github_det_easyconfig_paths_from_commit(self): args = [ os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), # absolute path 'toy-0.0-iter.eb', # relative path, available via robot search path - # commit in which ReFrame-4.3.2.eb was added, see https://github.com/easybuilders/easybuild-easyconfigs/pull/18763/commits + # commit in which ReFrame-4.3.2.eb was added, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18763/commits '--from-commit', commit, 'ReFrame-4.3.2.eb', # easyconfig included in commit, should be resolved via robot search path '--dry-run', From 529cf688c67a9cf45e9404d5239e071acc4bd2f0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 11 Mar 2024 10:42:43 +0100 Subject: [PATCH 153/430] make sure that full commit ID is provided to download_repo, and that tarball was actually downloaded + add extra test for download_repo with commit --- easybuild/tools/github.py | 16 ++++++++++++++-- test/framework/github.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c8058d8a8c..cdde9eb748 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -377,6 +377,15 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun mkdir(path, parents=True) if commit: + # make sure that full commit SHA-1 is provided + commit_sha1_regex = re.compile('[0-9a-f]{40}') + if commit_sha1_regex.match(commit): + _log.info("Valid commit SHA-1 provided for downloading %s/%s: %s", account, repo, commit) + else: + error_msg = "Specified commit SHA-1 %s for downloading %s/%s is not valid, " + error_msg += "must be full SHA-1 (40 chars)" + raise EasyBuildError(error_msg, commit, account, repo) + extracted_dir_name = '%s-%s' % (repo, commit) base_name = '%s.tar.gz' % commit latest_commit_sha = commit @@ -401,8 +410,11 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun target_path = os.path.join(path, base_name) _log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path)) - download_file(base_name, url, target_path, forced=True) - _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) + downloaded_path = download_file(base_name, url, target_path, forced=True) + if downloaded_path is None: + raise EasyBuildError("Failed to download tarball for %s/%s commit %s", account, repo, commit) + else: + _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) base_dir = extract_file(target_path, path, forced=True, change_into_dir=False) extracted_path = os.path.join(base_dir, extracted_dir_name) diff --git a/test/framework/github.py b/test/framework/github.py index f011c89b4f..868a653815 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -590,6 +590,34 @@ def test_github_download_repo(self): self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) self.assertExists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py')) + def test_github_download_repo_commit(self): + """Test downloading repo at specific commit (which does not require any GitHub token)""" + + # commit bdcc586189fcb3e5a340cddebb50d0e188c63cdc corresponds to easybuild-easyconfigs release v4.9.0 + test_commit = 'bdcc586189fcb3e5a340cddebb50d0e188c63cdc' + gh.download_repo(path=self.test_prefix, commit=test_commit) + repo_path = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-' + test_commit) + self.assertTrue(os.path.exists(repo_path)) + + setup_py_txt = read_file(os.path.join(repo_path, 'setup.py')) + self.assertTrue("VERSION = '4.9.0'" in setup_py_txt) + + # also check downloading non-default forked repo + test_commit = '434151c3dbf88b2382e8ead8655b4b2c01b92617' + gh.download_repo(path=self.test_prefix, account='boegel', repo='easybuild-framework', commit=test_commit) + repo_path = os.path.join(self.test_prefix, 'boegel', 'easybuild-framework-' + test_commit) + self.assertTrue(os.path.exists(repo_path)) + + release_notes_txt = read_file(os.path.join(repo_path, 'RELEASE_NOTES')) + self.assertTrue("v4.9.0 (30 December 2023)" in release_notes_txt) + + # short commit doesn't work, must be full commit ID + self.assertErrorRegex(EasyBuildError, "Specified commit SHA-1 bdcc586 .* is not valid", gh.download_repo, + path=self.test_prefix, commit='bdcc586') + + self.assertErrorRegex(EasyBuildError, "Failed to download tarball .* commit", gh.download_repo, + path=self.test_prefix, commit='0000000000000000000000000000000000000000') + def test_install_github_token(self): """Test for install_github_token function.""" if self.skip_github_tests: From a6fda46813f8069ba4ba7cb050300b65e15632f1 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 11 Mar 2024 07:30:53 -0400 Subject: [PATCH 154/430] Apply suggestions from code review Flip order of `is_prerelease` and version check for clarity. Co-authored-by: Kenneth Hoste --- easybuild/tools/modules.py | 6 +++--- test/framework/modules.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index ae358e3bbe..3fa735a6b8 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -254,7 +254,7 @@ def set_and_check_version(self): version = LooseVersion(self.version) if self.REQ_VERSION is not None: self.log.debug("Required minimum %s version defined: %s", self.NAME, self.REQ_VERSION) - if version.is_prerelease(self.REQ_VERSION, ['rc', '-beta']) or version < self.REQ_VERSION: + if version < self.REQ_VERSION or version.is_prerelease(self.REQ_VERSION, ['rc', '-beta']): raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s", self.NAME, self.REQ_VERSION, self.version) else: @@ -262,14 +262,14 @@ def set_and_check_version(self): if self.DEPR_VERSION is not None: self.log.debug("Deprecated %s version limit defined: %s", self.NAME, self.DEPR_VERSION) - if version.is_prerelease(self.DEPR_VERSION, ['rc', '-beta']) or version < self.DEPR_VERSION: + if version < self.DEPR_VERSION or version.is_prerelease(self.DEPR_VERSION, ['rc', '-beta']): depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) - if not version.is_prerelease(self.MAX_VERSION, ['rc', '-beta']) and self.version > self.MAX_VERSION: + if self.version > self.MAX_VERSION and not version.is_prerelease(self.MAX_VERSION, ['rc', '-beta']): raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s", self.NAME, self.MAX_VERSION, self.version) else: diff --git a/test/framework/modules.py b/test/framework/modules.py index efcad01d5a..dd6d66e70d 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -228,14 +228,14 @@ def test_avail(self): ms = self.modtool.available() version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_prerelease('5.7.5', ['rc']) and version >= '5.7.5': + if isinstance(self.modtool, Lmod) and version >= '5.7.5' and not version.is_prerelease('5.7.5', ['rc']): # with recent versions of Lmod, also the hidden modules are included in the output of 'avail' self.assertEqual(len(ms), TEST_MODULES_COUNT + 3) self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) - and not version.is_prerelease('4.6.0', ['-beta'])) and version >= '4.6.0': + and version >= '4.6.0' and not version.is_prerelease('4.6.0', ['-beta'])): # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) @@ -316,7 +316,7 @@ def test_exist(self): avail_mods = self.modtool.available() self.assertIn('Java/1.8.0_181', avail_mods) version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.0', ['rc']) and version >= '7.0': + if isinstance(self.modtool, Lmod) and version >= '7.0' and not version.is_prerelease('7.0', ['rc']): self.assertIn('Java/1.8', avail_mods) self.assertIn('Java/site_default', avail_mods) self.assertIn('JavaAlias', avail_mods) @@ -376,7 +376,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [True, True]) # also check with .modulerc.lua for Lmod 7.8 or newer - if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.8', ['rc']) and version >= '7.8': + if isinstance(self.modtool, Lmod) and version >= '7.8' and not version.is_prerelease('7.8', ['rc']): shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir) reset_module_caches() @@ -408,7 +408,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) # Test alias in home directory .modulerc - if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.0', ['rc']) or version >= '7.0': + if isinstance(self.modtool, Lmod) and version >= '7.0' and not version.is_prerelease('7.0', ['rc']): # Required or temporary HOME would be in MODULEPATH already self.init_testmods() # Sanity check: Module aliases don't exist yet From 7c71a8792a766604eaf4654dcf7972a3d72e8b02 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 13 Mar 2024 13:00:43 -0400 Subject: [PATCH 155/430] also run unit tests with Python 3.12 + add it to classifiers in setup.py --- .github/workflows/eb_command.yml | 2 +- .github/workflows/linting.yml | 2 +- .github/workflows/unit_tests.yml | 2 ++ setup.py | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index a41b50f572..74245f97b8 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 93af5bfc29..efd08662b6 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index a607ef43df..5698ece42c 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -41,6 +41,8 @@ jobs: modules_tool: ${{needs.setup.outputs.lmod8}} - python: '3.11' modules_tool: ${{needs.setup.outputs.lmod8}} + - python: '3.12' + modules_tool: ${{needs.setup.outputs.lmod8}} # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - python: 3.6 diff --git a/setup.py b/setup.py index c7b496867a..b8f8ae77af 100644 --- a/setup.py +++ b/setup.py @@ -116,6 +116,7 @@ def find_rel_test(): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Build Tools", ], platforms="Linux", From 1d4ede3971f9bf17f9c3a74073761fab4b766120 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 13 Mar 2024 13:32:54 -0400 Subject: [PATCH 156/430] Install setuptools for Python 3.12 (only) This (or "pip install build" with "python -m build") is needed for Python 3.12 with "python setup.py sdist" --- .github/workflows/eb_command.yml | 4 ++++ .github/workflows/unit_tests.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index 74245f97b8..ac909b5cd3 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -32,6 +32,10 @@ jobs: # update to latest pip, check version pip install --upgrade pip pip --version + if [[ "${{matrix.python}}" == 3.12 ]]; then + # needed for python setup.py sdist + pip install --upgrade setuptools + fi # for modules tool APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5698ece42c..60fdba402e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -85,6 +85,10 @@ jobs: pip install --upgrade pip pip --version pip install -r requirements.txt + if [[ "${{matrix.python}}" == 3.12 ]]; then + # needed for python setup.py sdist + pip install --upgrade setuptools + fi # git config is required to make actual git commits (cfr. tests for GitRepository) git config --global user.name "Travis CI" git config --global user.email "travis@travis-ci.org" From a7b4fb09bd51b05ab080467c72735a6e6428fb48 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 13 Mar 2024 21:45:47 -0400 Subject: [PATCH 157/430] Don't use "b" for strings with '\uxxxx'. This gives a SyntaxWarning in Python 3.12. In older Pythons it would not warn, but in all cases it gives a literal raw \u (with a backslash) instead of Unicode characters. Python 2.x needed u'\uxxxx' but the u is no longer needed with 3.x. --- test/framework/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 2f16e3978a..539523909f 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -153,7 +153,7 @@ def test_run_cmd(self): # this is constructed to reproduce errors like: # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 # UnicodeEncodeError: 'ascii' codec can't encode character u'\u2018' - for text in [b"foo \xe2 bar", b"foo \u2018 bar"]: + for text in [b"foo \xe2 bar", "foo \u2018 bar"]: test_file = os.path.join(self.test_prefix, 'foo.txt') write_file(test_file, text) cmd = "cat %s" % test_file @@ -182,7 +182,7 @@ def test_run_shell_cmd_basic(self): # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 # UnicodeEncodeError: 'ascii' codec can't encode character u'\u2018' # (such errors are ignored by the 'run' implementation) - for text in [b"foo \xe2 bar", b"foo \u2018 bar"]: + for text in [b"foo \xe2 bar", "foo \u2018 bar"]: test_file = os.path.join(self.test_prefix, 'foo.txt') write_file(test_file, text) cmd = "cat %s" % test_file From 38cc5ead11d98b7f28ddf97adf7f0b09e969fc6e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 14 Mar 2024 13:32:14 -0400 Subject: [PATCH 158/430] Use locale.getpreferredencoding(False) instead of explicit 'utf-8' This fixes Python 3.6 with LC_ALL=C. For Python 3.7+ we always get 'utf-8': https://docs.python.org/3/library/os.html#utf8-mode --- easybuild/tools/run.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 2592e198f1..ce2081d58d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -38,6 +38,7 @@ import contextlib import functools import inspect +import locale import os import re import signal @@ -350,8 +351,11 @@ def to_cmd_str(cmd): (stdout, stderr) = proc.communicate(input=stdin) # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out) - output = stdout.decode('utf-8', 'ignore') - stderr = stderr.decode('utf-8', 'ignore') if split_stderr else None + # getpreferredencoding normally gives 'utf-8' but can be ASCII (ANSI_X3.4-1968) + # for Python 3.6 and older with LC_ALL=C + encoding = locale.getpreferredencoding(False) + output = stdout.decode(encoding, 'ignore') + stderr = stderr.decode(encoding, 'ignore') if split_stderr else None # store command output to temporary file(s) if output_file: From 3e117fec6efb65f26b459dd272f263f9b76e144e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 16 Mar 2024 10:47:35 +0100 Subject: [PATCH 159/430] add tests for fetch_files_from_commit and fetch_easyconfigs_from_commit --- easybuild/tools/github.py | 22 +++++++++----- test/framework/github.py | 64 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index cdde9eb748..2f67285e1b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -377,12 +377,12 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun mkdir(path, parents=True) if commit: - # make sure that full commit SHA-1 is provided + # make sure that full commit SHA is provided commit_sha1_regex = re.compile('[0-9a-f]{40}') if commit_sha1_regex.match(commit): - _log.info("Valid commit SHA-1 provided for downloading %s/%s: %s", account, repo, commit) + _log.info("Valid commit SHA provided for downloading %s/%s: %s", account, repo, commit) else: - error_msg = "Specified commit SHA-1 %s for downloading %s/%s is not valid, " + error_msg = r"Specified commit SHA %s for downloading %s/%s is not valid, " error_msg += "must be full SHA-1 (40 chars)" raise EasyBuildError(error_msg, commit, account, repo) @@ -622,6 +622,8 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, else: raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s", commit, extra_ec_paths) + else: + path = os.path.join(tempfile.gettempdir(), 'ecs_commit_' + commit) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_commit_' + commit) @@ -633,12 +635,14 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff') diff_fn = os.path.basename(diff_url) diff_filepath = os.path.join(path, diff_fn) - download_file(diff_fn, diff_url, diff_filepath, forced=True) - diff_txt = read_file(diff_filepath) - _log.debug("Diff for commit %s:\n%s", commit, diff_txt) + if download_file(diff_fn, diff_url, diff_filepath, forced=True): + diff_txt = read_file(diff_filepath) + _log.debug("Diff for commit %s:\n%s", commit, diff_txt) - files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True) - _log.debug("List of patched files for commit %s: %s", commit, files) + files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True) + _log.debug("List of patched files for commit %s: %s", commit, files) + else: + raise EasyBuildError("Failed to download diff for commit %s of %s/%s", commit, github_account, github_repo) # download tarball for specific commit repo_commit = download_repo(repo=github_repo, commit=commit, account=github_account) @@ -647,6 +651,8 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, files_subdir = 'easybuild/easyconfigs/' elif github_repo == GITHUB_EASYBLOCKS_REPO: files_subdir = 'easybuild/easyblocks/' + else: + raise EasyBuildError("Unknown repo: %s" % github_repo) # copy specified files to directory where they're expected to be found file_paths = [] diff --git a/test/framework/github.py b/test/framework/github.py index 868a653815..4bcf9b8987 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -50,7 +50,8 @@ from easybuild.tools.filetools import read_file, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_MERGEABLE_STATE_CLEAN from easybuild.tools.github import VALID_CLOSE_PR_REASONS -from easybuild.tools.github import det_pr_title, is_patch_for, pick_default_branch +from easybuild.tools.github import det_pr_title, fetch_easyconfigs_from_commit, fetch_files_from_commit +from easybuild.tools.github import is_patch_for, pick_default_branch from easybuild.tools.testing import create_test_report, post_pr_test_report, session_state from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters import easybuild.tools.github as gh @@ -536,6 +537,65 @@ def test_github_fetch_files_from_pr_cache(self): res = gh.fetch_easyblocks_from_pr(12345, tmpdir) self.assertEqual(sorted(pr12345_files), sorted(res)) + def test_fetch_files_from_commit(self): + """Test fetch_files_from_commit function.""" + + # easyconfigs commit to add EasyBuild-4.8.2.eb (also test short commit, should work) + test_commit = '7c83a553950c233943c7b0189762f8c05cfea852' + + # without specifying any files/repo, default is to use easybuilders/easybuilld-easyconfigs + # and determine which files were changed in the commit + res = fetch_files_from_commit(test_commit) + self.assertEqual(len(res), 1) + ec_path = res[0] + expected_path = 'ecs_commit_7c83a553950c233943c7b0189762f8c05cfea852/e/EasyBuild/EasyBuild-4.8.2.eb' + self.assertTrue(ec_path.endswith(expected_path)) + self.assertTrue(os.path.exists(ec_path)) + self.assertIn("version = '4.8.2'", read_file(ec_path)) + + # also test downloading a specific file from easyblocks repo + # commit that enables use_pip & co in PythonPackage easyblock + test_commit = 'd6f0cd7b586108e40f7cf1f1054bb07e16718caf' + res = fetch_files_from_commit(test_commit, files=['pythonpackage.py'], + github_account='easybuilders', github_repo='easybuild-easyblocks') + self.assertEqual(len(res), 1) + self.assertIn("'use_pip': [True,", read_file(res[0])) + + # test downloading with short commit, download_repo currently enforces using long commit + error_pattern = r"Specified commit SHA 7c83a55 for downloading easybuilders/easybuild-easyconfigs " + error_pattern += r"is not valid, must be full SHA-1 \(40 chars\)" + self.assertErrorRegex(EasyBuildError, error_pattern, fetch_files_from_commit, '7c83a55') + + # test downloading of non-existing commit + error_pattern = r"Failed to download diff for commit c0ff33c0ff33 of easybuilders/easybuild-easyconfigs" + self.assertErrorRegex(EasyBuildError, error_pattern, fetch_files_from_commit, 'c0ff33c0ff33') + + def test_fetch_easyconfigs_from_commit(self): + """Test fetch_easyconfigs_from_commit function.""" + + # commit in which easyconfigs for PyTables 3.9.2 + dependencies were added + test_commit = '6515b44cd84a20fe7876cb4bdaf3c0080e688566' + + # without specifying any files/repo, default is to determine which files were changed in the commit + res = fetch_easyconfigs_from_commit(test_commit) + self.assertEqual(len(res), 5) + expected_ec_filenames = ['Blosc-1.21.5-GCCcore-13.2.0.eb', 'Blosc2-2.13.2-GCCcore-13.2.0.eb', + 'PyTables-3.9.2-foss-2023b.eb', 'PyTables-3.9.2_fix-find-blosc2-dep.patch', + 'py-cpuinfo-9.0.0-GCCcore-13.2.0.eb'] + self.assertEqual(sorted([os.path.basename(f) for f in res]), expected_ec_filenames) + for ec_path in res: + self.assertTrue(os.path.exists(ec_path)) + if ec_path.endswith('.eb'): + self.assertIn("version =", read_file(ec_path)) + else: + self.assertTrue(ec_path.endswith('.patch')) + + # merge commit for release of EasyBuild v4.9.0 + test_commit = 'bdcc586189fcb3e5a340cddebb50d0e188c63cdc' + res = fetch_easyconfigs_from_commit(test_commit, files=['RELEASE_NOTES'], path=self.test_prefix) + self.assertEqual(len(res), 1) + self.assertIn("v4.9.0 (30 December 2023)", read_file(res[0])) + def test_github_fetch_latest_commit_sha(self): """Test fetch_latest_commit_sha function.""" if self.skip_github_tests: @@ -612,7 +672,7 @@ def test_github_download_repo_commit(self): self.assertTrue("v4.9.0 (30 December 2023)" in release_notes_txt) # short commit doesn't work, must be full commit ID - self.assertErrorRegex(EasyBuildError, "Specified commit SHA-1 bdcc586 .* is not valid", gh.download_repo, + self.assertErrorRegex(EasyBuildError, "Specified commit SHA bdcc586 .* is not valid", gh.download_repo, path=self.test_prefix, commit='bdcc586') self.assertErrorRegex(EasyBuildError, "Failed to download tarball .* commit", gh.download_repo, From b6e301898c364aeee1840bb066447cec7ccb18a1 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 18 Mar 2024 07:34:11 -0400 Subject: [PATCH 160/430] Explicitly check if `python -c "import distutils"` is possible This makes the check more general and compatible with later versions of Python than just 3.12. --- .github/workflows/eb_command.yml | 4 ++-- .github/workflows/unit_tests.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index ac909b5cd3..ec99594a86 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -32,8 +32,8 @@ jobs: # update to latest pip, check version pip install --upgrade pip pip --version - if [[ "${{matrix.python}}" == 3.12 ]]; then - # needed for python setup.py sdist + if ! python -c "import distutils" 2> /dev/null; then + # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist pip install --upgrade setuptools fi diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 60fdba402e..ce739ff46d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -85,8 +85,8 @@ jobs: pip install --upgrade pip pip --version pip install -r requirements.txt - if [[ "${{matrix.python}}" == 3.12 ]]; then - # needed for python setup.py sdist + if ! python -c "import distutils" 2> /dev/null; then + # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist pip install --upgrade setuptools fi # git config is required to make actual git commits (cfr. tests for GitRepository) From f5489bf2780256291a785e22bbdfe2c05c638407 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 24 Mar 2024 11:38:01 +0100 Subject: [PATCH 161/430] add test for --from-commit --- test/framework/options.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 26daa0cc7a..6cd46de021 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2011,6 +2011,44 @@ def test_github_from_pr_x(self): except URLError as err: print("Ignoring URLError '%s' in test_from_pr_x" % err) + def test_from_commit(self): + """Test for --from-commit.""" + # note: --from-commit does not involve using GitHub API, so no GitHub token required + + # easyconfigs commit to add EasyBuild-4.8.2.eb + test_commit = '7c83a553950c233943c7b0189762f8c05cfea852' + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + tmpdir = tempfile.mkdtemp() + args = [ + '--from-commit=%s' % test_commit, + '--dry-run', + '--tmpdir=%s' % tmpdir, + ] + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (tmpdir, 'EasyBuild/4.8.2'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + path = '.*%s' % os.path.dirname(path_prefix) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + + pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_commit_%s' % test_commit) + regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + except URLError as err: + print("Ignoring URLError '%s' in test_from_pr" % err) + shutil.rmtree(tmpdir) + def test_no_such_software(self): """Test using no arguments.""" From fd7b9f9fcadb860ab0f26e11f10eee3ef14e1789 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 27 Mar 2024 14:18:24 +0100 Subject: [PATCH 162/430] add --short option to findUpdatedEcs This is useful if you want to find & copy the names of ECs to rebuild. Also fixed indentation (4 spaces consistently) and use a unified diff. --- easybuild/scripts/findUpdatedEcs.sh | 58 ++++++++++++++++------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh index d6631bfe0b..66b7e34bd8 100755 --- a/easybuild/scripts/findUpdatedEcs.sh +++ b/easybuild/scripts/findUpdatedEcs.sh @@ -33,51 +33,57 @@ function checkModule { first_letter=${ec_filename:0:1} letterPath=$easyconfigFolder/${first_letter,,} if [[ -d "$letterPath" ]]; then - ec_new="$(find "$letterPath" -type f -name "$ec_filename")" + ec_new="$(find "$letterPath" -type f -name "$ec_filename")" else - ec_new= + ec_new= fi # Fallback if not found [[ -n "$ec_new" ]] || ec_new="$(find "$easyconfigFolder" -type f -name "$ec_filename")" if [[ -z "$ec_new" ]]; then - printError "=== Did not find new EC $ec_filename" + printError "=== Did not find new EC $ec_filename" elif [[ ! -e "$ec_new" ]]; then printError "=== Found multiple new ECs: $ec_new" - elif ! out=$(diff "$ec_installed" "$ec_new"); then - echo -e "${YELLOW}=== Needs updating: ${GREEN}${ec_installed}${YELLOW} vs ${GREEN}${ec_new}${NC}" - if ((showDiff == 1)); then - echo "$out" + elif ! out=$(diff -u "$ec_installed" "$ec_new"); then + if ((short == 1)); then + basename "$ec_installed" + else + echo -e "${YELLOW}=== Needs updating: ${GREEN}${ec_installed}${YELLOW} vs ${GREEN}${ec_new}${NC}" + if ((showDiff == 1)); then + echo "$out" + fi fi fi } ecDefaultFolder= if path=$(which eb 2>/dev/null); then - path=$(dirname "$path") - for p in "$path" "$(dirname "$path")"; do - if [ -d "$p/easybuild/easyconfigs" ]; then - ecDefaultFolder=$p - break - fi - done + path=$(dirname "$path") + for p in "$path" "$(dirname "$path")"; do + if [ -d "$p/easybuild/easyconfigs" ]; then + ecDefaultFolder=$p + break + fi + done fi function usage { - echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER" - echo - echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed." - echo "Can either check the currently loaded modules or all modules installed in a specific location" - echo - echo "--verbose Verbose status output while checking" - echo "--loaded Check only currently loaded modules" - echo "--diff Show diff of changed module files" - echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries" - echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}" - exit 0 + echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER" + echo + echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed." + echo "Can either check the currently loaded modules or all modules installed in a specific location" + echo + echo "--verbose Verbose status output while checking" + echo "--loaded Check only currently loaded modules" + echo "--short Only show filename of changed ECs" + echo "--diff Show diff of changed module files" + echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries" + echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}" + exit 0 } checkLoadedModules=0 showDiff=0 +short=0 modulesFolder="" easyconfigFolder=$ecDefaultFolder @@ -89,6 +95,8 @@ while [[ $# -gt 0 ]]; do verbose=1;; -d|--diff) showDiff=1;; + -s|--short) + short=1;; -l|--loaded) checkLoadedModules=1;; -m|--modules) From 162aad0e7f1b8a27dd22a6e6bb198ef5af9367ee Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Mar 2024 21:36:13 +0100 Subject: [PATCH 163/430] implement support for --include-easyblocks-from-commit --- easybuild/tools/config.py | 1 + easybuild/tools/github.py | 7 ++- easybuild/tools/options.py | 110 +++++++++++++++++++++++-------------- easybuild/tools/testing.py | 8 +++ 4 files changed, 85 insertions(+), 41 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ebc4f4a7d1..21df1f1e91 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -230,6 +230,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'http_header_fields_urlpat', 'hooks', 'ignore_dirs', + 'include_easyblocks_from_commit', 'insecure_download', 'job_backend_config', 'job_cores', diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 2f67285e1b..eda237fca1 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -590,7 +590,7 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi def fetch_easyblocks_from_pr(pr, path=None, github_user=None): - """Fetch patched easyconfig files for a particular PR.""" + """Fetch patched easyblocks for a particular PR.""" return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) @@ -685,6 +685,11 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, return file_paths +def fetch_easyblocks_from_commit(commit, files=None, path=None): + """Fetch easyblocks from a specified commit.""" + return fetch_files_from_commit(commit, files=files, path=path, github_repo=GITHUB_EASYBLOCKS_REPO) + + def fetch_easyconfigs_from_commit(commit, files=None, path=None): """Fetch specified easyconfig files from a specific commit.""" return fetch_files_from_commit(commit, files=files, path=path, github_repo=GITHUB_EASYCONFIGS_REPO) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index f99a5ac284..f9093ecc94 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -89,7 +89,7 @@ from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS -from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token +from easybuild.tools.github import fetch_easyblocks_from_commit, fetch_easyblocks_from_pr, fetch_github_token from easybuild.tools.hooks import KNOWN_HOOKS from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains from easybuild.tools.job.backend import avail_job_backends @@ -702,11 +702,13 @@ def github_options(self): 'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False), 'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), - 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', None, {'metavar': 'PR#'}), + 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', None, {'metavar': 'commit_SHA'}), 'from-pr': ("Obtain easyconfigs from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None), 'github-user': ("GitHub username", str, 'store', None), 'github-org': ("GitHub organization", str, 'store', None), + 'include-easyblocks-from-commit': ("Include easyblocks from specified commit", 'str', 'store', None, + {'metavar': 'commit_SHA'}), 'include-easyblocks-from-pr': ("Include easyblocks from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'install-github-token': ("Install GitHub token (requires --github-user)", None, 'store_true', False), @@ -1225,8 +1227,9 @@ def _postprocess_list_avail(self): if self.options.avail_easyconfig_licenses: msg += avail_easyconfig_licenses(self.options.output_format) - # dump available easyblocks (unless including easyblocks from pr, in which case it will be done later) - if self.options.list_easyblocks and not self.options.include_easyblocks_from_pr: + # dump available easyblocks (unless including easyblocks from commit or PR, in which case it will be done later) + easyblocks_from = self.options.include_easyblocks_from_commit or self.options.include_easyblocks_from_pr + if self.options.list_easyblocks and not easyblocks_from: msg += list_easyblocks(self.options.list_easyblocks, self.options.output_format) # dump known toolchains @@ -1270,7 +1273,7 @@ def _postprocess_list_avail(self): print(msg) # cleanup tmpdir and exit - if not self.options.include_easyblocks_from_pr: + if not (self.options.include_easyblocks_from_commit or self.options.include_easyblocks_from_pr): cleanup_and_exit(self.tmpdir) def avail_repositories(self): @@ -1524,6 +1527,67 @@ def check_root_usage(allow_use_as_root=False): raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here.") +def handle_include_easyblocks_from(options, log): + """ + Handle --include-easyblocks-from-pr and --include-easyblocks-from-commit + """ + def check_included_multiple(included_easyblocks_from, source): + """Check whether easyblock is being included multiple times""" + included_multiple = included_easyblocks_from & included_easyblocks + if included_multiple: + warning_msg = "One or more easyblocks included from multiple locations: %s " \ + % ', '.join(included_multiple) + warning_msg += "(the one(s) from %s will be used)" % source + print_warning(warning_msg) + + if options.include_easyblocks_from_pr or options.include_easyblocks_from_commit: + + if options.include_easyblocks: + # check if you are including the same easyblock twice + included_paths = expand_glob_paths(options.include_easyblocks) + included_easyblocks = set([os.path.basename(eb) for eb in included_paths]) + + if options.include_easyblocks_from_pr: + try: + easyblock_prs = [int(x) for x in options.include_easyblocks_from_pr] + except ValueError: + raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") + + for easyblock_pr in easyblock_prs: + easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr) + included_from_pr = set([os.path.basename(eb) for eb in easyblocks_from_pr]) + + if options.include_easyblocks: + check_included_multiple(included_from_pr, "PR #%s" % easyblock_pr) + + included_easyblocks |= included_from_pr + + for easyblock in included_from_pr: + print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log) + + include_easyblocks(options.tmpdir, easyblocks_from_pr) + + easyblock_commit = options.include_easyblocks_from_commit + if easyblock_commit: + easyblocks_from_commit = fetch_easyblocks_from_commit(easyblock_commit) + included_from_commit = set([os.path.basename(eb) for eb in easyblocks_from_commit]) + + if options.include_easyblocks: + check_included_multiple(included_from_commit, "commit %s" % easyblock_commit) + + for easyblock in included_from_commit: + print_msg("easyblock %s included from comit %s" % (easyblock, easyblock_commit), log=log) + + include_easyblocks(options.tmpdir, easyblocks_from_commit) + + if options.list_easyblocks: + msg = list_easyblocks(options.list_easyblocks, options.output_format) + if options.unittest_file: + log.info(msg) + else: + print(msg) + cleanup_and_exit(tmpdir) + def set_up_configuration(args=None, logfile=None, testing=False, silent=False, reconfigure=False): """ @@ -1633,41 +1697,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r init_build_options(build_options=build_options, cmdline_options=options) # done here instead of in _postprocess_include because github integration requires build_options to be initialized - if eb_go.options.include_easyblocks_from_pr: - try: - easyblock_prs = [int(x) for x in eb_go.options.include_easyblocks_from_pr] - except ValueError: - raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") - - if eb_go.options.include_easyblocks: - # check if you are including the same easyblock twice - included_paths = expand_glob_paths(eb_go.options.include_easyblocks) - included_from_file = set([os.path.basename(eb) for eb in included_paths]) - - for easyblock_pr in easyblock_prs: - easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr) - included_from_pr = set([os.path.basename(eb) for eb in easyblocks_from_pr]) - - if eb_go.options.include_easyblocks: - included_twice = included_from_pr & included_from_file - if included_twice: - warning_msg = "One or more easyblocks included from multiple locations: %s " \ - % ', '.join(included_twice) - warning_msg += "(the one(s) from PR #%s will be used)" % easyblock_pr - print_warning(warning_msg) - - for easyblock in included_from_pr: - print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log) - - include_easyblocks(eb_go.options.tmpdir, easyblocks_from_pr) - - if eb_go.options.list_easyblocks: - msg = list_easyblocks(eb_go.options.list_easyblocks, eb_go.options.output_format) - if eb_go.options.unittest_file: - log.info(msg) - else: - print(msg) - cleanup_and_exit(tmpdir) + handle_include_easyblocks_from(eb_go.options, log) check_python_version() diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index ffd8ce580b..a58e539c23 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -336,6 +336,14 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state, comment_lines = ["Test report by @%s" % github_user] + if build_option('include_easyblocks_from_commit'): + if repo_type == GITHUB_EASYCONFIGS_REPO: + easyblocks_commit = build_option('include_easyblocks_from_commit') + url = 'https://github.com/%s/%s/commit/%s' % (pr_target_account, GITHUB_EASYBLOCKS_REPO, easyblocks_commit) + comment_lines.append("Using easyblocks from %s" % url) + else: + raise EasyBuildError("Don't know how to submit test reports to repo %s.", repo_type) + if build_option('include_easyblocks_from_pr'): if repo_type == GITHUB_EASYCONFIGS_REPO: easyblocks_pr_nrs = [int(x) for x in build_option('include_easyblocks_from_pr')] From 44f94c22fbfbcd230d83ae133b509ee2a26cb9e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Mar 2024 09:54:01 +0100 Subject: [PATCH 164/430] define flags for --optarch=GENERIC on RISC-V --- easybuild/toolchains/compiler/clang.py | 1 + easybuild/toolchains/compiler/gcc.py | 1 + 2 files changed, 2 insertions(+) diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 303f0ed0e4..4ab46abde7 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -98,6 +98,7 @@ class Clang(Compiler): } # used with --optarch=GENERIC COMPILER_GENERIC_OPTION = { + (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # the default for -mabi is system-dependent (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic', (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic', } diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 3fba8d3954..84a849d596 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -94,6 +94,7 @@ class Gcc(Compiler): (systemtools.AARCH64, systemtools.ARM): 'mcpu=generic', # implies -march=armv8-a and -mtune=generic (systemtools.POWER, systemtools.POWER): 'mcpu=powerpc64', # no support for -march on POWER (systemtools.POWER, systemtools.POWER_LE): 'mcpu=powerpc64le', # no support for -march on POWER + (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # the default for -mabi is system-dependent (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic', (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic', } From ab93030fb9831426a1ac6515623cf68c686852ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Mar 2024 10:23:56 +0100 Subject: [PATCH 165/430] add additional space, shorten line --- easybuild/toolchains/compiler/clang.py | 2 +- easybuild/toolchains/compiler/gcc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 4ab46abde7..8c93053932 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -98,7 +98,7 @@ class Clang(Compiler): } # used with --optarch=GENERIC COMPILER_GENERIC_OPTION = { - (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # the default for -mabi is system-dependent + (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic', (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic', } diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 84a849d596..fd20846592 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -94,7 +94,7 @@ class Gcc(Compiler): (systemtools.AARCH64, systemtools.ARM): 'mcpu=generic', # implies -march=armv8-a and -mtune=generic (systemtools.POWER, systemtools.POWER): 'mcpu=powerpc64', # no support for -march on POWER (systemtools.POWER, systemtools.POWER_LE): 'mcpu=powerpc64le', # no support for -march on POWER - (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # the default for -mabi is system-dependent + (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic', (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic', } From f2ed0ff9f9184983579f7815ce3be473536b34e7 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 31 Mar 2024 11:06:54 +0100 Subject: [PATCH 166/430] hide readelf output --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index aa77793929..9f7ac535a7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3187,7 +3187,7 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True): # check whether RPATH section in 'readelf -d' output is there if check_readelf_rpath: fail_msg = None - res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False) + res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False, hidden=True) if res.exit_code: fail_msg = f"Failed to run 'readelf -d {path}': {res.output}" elif not readelf_rpath_regex.search(res.output): From f8c683b6140c40dd47b74b0291b2165a89c44ad6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 31 Mar 2024 11:45:34 +0100 Subject: [PATCH 167/430] switch from to `bash` in tests --- test/framework/filetools.py | 2 +- test/framework/systemtools.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 91afdf79f1..db85575cdb 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -924,7 +924,7 @@ def test_is_binary(self): self.assertTrue(ft.is_binary(b'\00')) self.assertTrue(ft.is_binary(b"File is binary when it includes \00 somewhere")) - self.assertTrue(ft.is_binary(ft.read_file('/bin/ls', mode='rb'))) + self.assertTrue(ft.is_binary(ft.read_file('/bin/bash', mode='rb'))) def test_det_patched_files(self): """Test det_patched_files function.""" diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 6d8395a5fc..68a81c814f 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1033,15 +1033,15 @@ def test_check_linked_shared_libs(self): self.assertEqual(check_linked_shared_libs(txt_path), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path), None) - bin_ls_path = which('ls') + bin_bash_path = which('bash') os_type = get_os_type() if os_type == LINUX: with self.mocked_stdout_stderr(): - res = run_shell_cmd("ldd %s" % bin_ls_path) + res = run_shell_cmd("ldd %s" % bin_bash_path) elif os_type == DARWIN: with self.mocked_stdout_stderr(): - res = run_shell_cmd("otool -L %s" % bin_ls_path) + res = run_shell_cmd("otool -L %s" % bin_bash_path) else: raise EasyBuildError("Unknown OS type: %s" % os_type) @@ -1061,7 +1061,7 @@ def test_check_linked_shared_libs(self): self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) - for path in (bin_ls_path, lib_path): + for path in (bin_bash_path, lib_path): # path may not exist, especially for library paths obtained via 'otool -L' on macOS if os.path.exists(path): error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args) @@ -1078,7 +1078,7 @@ def test_check_linked_shared_libs(self): self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) - for path in (bin_ls_path, lib_path): + for path in (bin_bash_path, lib_path): error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args) self.assertFalse(check_linked_shared_libs(path, **pattern_named_args), error_msg) From ba306a4966d24f3e89acc31e87bbe48ad83fe638 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Apr 2024 21:30:54 +0200 Subject: [PATCH 168/430] fix use of 'tmpdir' in handle_include_easyblocks_from + trivial style fixes --- easybuild/tools/options.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index f9093ecc94..91bede9553 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -702,7 +702,8 @@ def github_options(self): 'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False), 'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), - 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', None, {'metavar': 'commit_SHA'}), + 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', + None, {'metavar': 'commit_SHA'}), 'from-pr': ("Obtain easyconfigs from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None), 'github-user': ("GitHub username", str, 'store', None), @@ -1527,6 +1528,7 @@ def check_root_usage(allow_use_as_root=False): raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here.") + def handle_include_easyblocks_from(options, log): """ Handle --include-easyblocks-from-pr and --include-easyblocks-from-commit @@ -1551,7 +1553,7 @@ def check_included_multiple(included_easyblocks_from, source): try: easyblock_prs = [int(x) for x in options.include_easyblocks_from_pr] except ValueError: - raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") + raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s") for easyblock_pr in easyblock_prs: easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr) @@ -1573,7 +1575,7 @@ def check_included_multiple(included_easyblocks_from, source): included_from_commit = set([os.path.basename(eb) for eb in easyblocks_from_commit]) if options.include_easyblocks: - check_included_multiple(included_from_commit, "commit %s" % easyblock_commit) + check_included_multiple(included_from_commit, "commit %s" % easyblock_commit) for easyblock in included_from_commit: print_msg("easyblock %s included from comit %s" % (easyblock, easyblock_commit), log=log) @@ -1586,6 +1588,8 @@ def check_included_multiple(included_easyblocks_from, source): log.info(msg) else: print(msg) + # tmpdir is set by option parser via set_tmpdir function + tmpdir = tempfile.gettempdir() cleanup_and_exit(tmpdir) From 799989a202a59e29d71dd4d825afbf26e7d96a77 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Apr 2024 21:56:50 +0200 Subject: [PATCH 169/430] add test for --include-easyblocks-from-commit --- test/framework/options.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 6cd46de021..3b9a46d0f5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2028,6 +2028,7 @@ def test_from_commit(self): '--tmpdir=%s' % tmpdir, ] try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) modules = [ (tmpdir, 'EasyBuild/4.8.2'), @@ -2046,7 +2047,43 @@ def test_from_commit(self): regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) except URLError as err: - print("Ignoring URLError '%s' in test_from_pr" % err) + print("Ignoring URLError '%s' in test_from_commit" % err) + shutil.rmtree(tmpdir) + + def test_include_easyblocks_from_commit(self): + """Test for --include-easyblocks-from-commit.""" + # note: --include-easyblocks-from-commit does not involve using GitHub API, so no GitHub token required + + # easyblocks commit only touching ConfigureMake easyblock + test_commit = '6005d37a1ed2b130a18b5bd525df810f19ba3bbd' + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + tmpdir = tempfile.mkdtemp() + args = [ + '--include-easyblocks-from-commit=%s' % test_commit, + '--dry-run', + '--tmpdir=%s' % tmpdir, + 'toy-0.0.eb', # test easyconfig + ] + try: + self.mock_stdout(True) + self.mock_stderr(True) + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + pattern = "== easyblock configuremake.py included from comit %s" % test_commit + self.assertEqual(stderr, '') + self.assertIn(pattern, stdout) + + regex = re.compile(r"^ \* \[.\] .*/toy-0.0.eb \(module: toy/0.0\)$", re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + except URLError as err: + print("Ignoring URLError '%s' in test_include_easyblocks_from_commit" % err) shutil.rmtree(tmpdir) def test_no_such_software(self): From de5e2350102eaa4792db4edf7bf6098605e514f2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Apr 2024 22:32:34 +0200 Subject: [PATCH 170/430] fix combination of --copy-ec and --from-commit --- easybuild/framework/easyconfig/tools.py | 41 +++++++++++++++- easybuild/main.py | 2 +- test/framework/options.py | 65 ++++++++++++++++++++++++- 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 711e14d29e..7737d72860 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -62,7 +62,7 @@ from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data -from easybuild.tools.github import fetch_files_from_pr +from easybuild.tools.github import fetch_files_from_commit, fetch_files_from_pr from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -791,7 +791,7 @@ def avail_easyblocks(): return easyblocks -def det_copy_ec_specs(orig_paths, from_pr): +def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None): """Determine list of paths + target directory for --copy-ec.""" if from_pr is not None and not isinstance(from_pr, list): @@ -855,4 +855,41 @@ def det_copy_ec_specs(orig_paths, from_pr): elif pr_matches: raise EasyBuildError("Found multiple paths for %s in PR: %s", filename, pr_matches) + # consider --from-commit (only if --from-pr was not used) + elif from_commit: + tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_commit_%s' % from_commit) + commit_paths = fetch_files_from_commit(from_commit, path=tmpdir) + + # assume that files need to be copied to current working directory for now + target_path = os.getcwd() + + if orig_paths: + last_path = orig_paths[-1] + + # check files touched by commit and see if the target directory for --copy-ec + # corresponds to the name of one of these files; + # if so we should copy the specified file(s) to the current working directory, + # since interpreting the last argument as target location is very unlikely to be correct in this case + commit_filenames = [os.path.basename(p) for p in commit_paths] + if last_path in commit_filenames: + paths = orig_paths[:] + else: + target_path = last_path + # exclude last argument that is used as target location + paths = orig_paths[:-1] + + # if list of files to copy is empty at this point, + # we simply copy *all* files touched by the PR + if not paths: + paths = commit_paths + + # replace path for files touched by commit (no need to worry about others) + for idx, path in enumerate(paths): + filename = os.path.basename(path) + commit_matches = [x for x in commit_paths if os.path.basename(x) == filename] + if len(commit_matches) == 1: + paths[idx] = commit_matches[0] + elif commit_matches: + raise EasyBuildError("Found multiple paths for %s in commit: %s", filename, commit_matches) + return paths, target_path diff --git a/easybuild/main.py b/easybuild/main.py index 85501f57f0..4ca7bcbb95 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -329,7 +329,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session if options.copy_ec: # figure out list of files to copy + target location (taking into account --from-pr) - eb_args, target_path = det_copy_ec_specs(eb_args, from_pr_list) + eb_args, target_path = det_copy_ec_specs(eb_args, from_pr=from_pr_list, from_commit=options.from_commit) categorized_paths = categorize_files_by_type(eb_args) diff --git a/test/framework/options.py b/test/framework/options.py index 3b9a46d0f5..623aff63f6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1422,6 +1422,70 @@ def test_github_copy_ec_from_pr(self): self.assertIn("name = 'ExifTool'", read_file(test_ec)) remove_file(test_ec) + def test_copy_ec_from_commit(self): + """Test combination of --copy-ec with --from-commit.""" + # note: --from-commit does not involve using GitHub API, so no GitHub token required + + # using easyconfigs commit to add EasyBuild-4.8.2.eb + test_commit = '7c83a553950c233943c7b0189762f8c05cfea852' + + test_dir = os.path.join(self.test_prefix, 'from_commit') + mkdir(test_dir, parents=True) + args = ['--copy-ec', '--from-commit=%s' % test_commit, test_dir] + try: + stdout = self.mocked_main(args) + except URLError as err: + print("Ignoring URLError '%s' in test_copy_ec_from_commit" % err) + + pattern = "_%s/e/EasyBuild/EasyBuild-4.8.2.eb copied to " % test_commit + self.assertIn(pattern, stdout) + copied_ecs = os.listdir(test_dir) + self.assertEqual(copied_ecs, ['EasyBuild-4.8.2.eb']) + + # cleanup + remove_dir(test_dir) + mkdir(test_dir) + + # test again, using extra argument (name of file to copy), without specifying target directory + # (should copy to current directory) + cwd = change_dir(test_dir) + args = ['--copy-ec', '--from-commit=%s' % test_commit, "EasyBuild-4.8.2.eb"] + try: + stdout = self.mocked_main(args) + except URLError as err: + print("Ignoring URLError '%s' in test_copy_ec_from_commit" % err) + + self.assertIn(pattern, stdout) + copied_ecs = os.listdir(test_dir) + self.assertEqual(copied_ecs, ['EasyBuild-4.8.2.eb']) + + # cleanup + change_dir(cwd) + remove_dir(test_dir) + mkdir(test_dir) + + # test with commit that touches a bunch of easyconfigs + test_commit = '49c887397b1a948e1909fc24bc905fdc1ad38388' + expected_ecs = [ + 'gompi-2023b.eb', + 'gfbf-2023b.eb', + 'ScaLAPACK-2.2.0-gompi-2023b-fb.eb', + 'foss-2023b.eb', + 'HPL-2.3-foss-2023b.eb', + 'FFTW.MPI-3.3.10-gompi-2023b.eb', + 'SciPy-bundle-2023.11-gfbf-2023b.eb', + 'OSU-Micro-Benchmarks-7.2-gompi-2023b.eb', + ] + args = ['--copy-ec', '--from-commit=%s' % test_commit, test_dir] + try: + stdout = self.mocked_main(args) + except URLError as err: + print("Ignoring URLError '%s' in test_copy_ec_from_commit" % err) + + copied_ecs = os.listdir(test_dir) + for ec in expected_ecs: + self.assertIn(ec, copied_ecs) + def test_dry_run(self): """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -2028,7 +2092,6 @@ def test_from_commit(self): '--tmpdir=%s' % tmpdir, ] try: - outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) modules = [ (tmpdir, 'EasyBuild/4.8.2'), From 90c4939396e1d8e8a0b9d5649dddca721e3db346 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 09:11:09 +0200 Subject: [PATCH 171/430] use different commit for testing --include-easyblocks-from-commit + clean up imported easyblock --- test/framework/options.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 623aff63f6..8b578d74da 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2113,12 +2113,15 @@ def test_from_commit(self): print("Ignoring URLError '%s' in test_from_commit" % err) shutil.rmtree(tmpdir) - def test_include_easyblocks_from_commit(self): + # must be run after test for --list-easyblocks, hence the '_xxx_' + # cleaning up the imported easyblocks is quite difficult... + def test_xxx_include_easyblocks_from_commit(self): """Test for --include-easyblocks-from-commit.""" # note: --include-easyblocks-from-commit does not involve using GitHub API, so no GitHub token required - # easyblocks commit only touching ConfigureMake easyblock - test_commit = '6005d37a1ed2b130a18b5bd525df810f19ba3bbd' + orig_local_sys_path = sys.path[:] + # easyblocks commit only touching Binary easyblock + test_commit = '94d28c556947bd96d0978df775b15a50a4600c6f' fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -2138,7 +2141,16 @@ def test_include_easyblocks_from_commit(self): stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) - pattern = "== easyblock configuremake.py included from comit %s" % test_commit + + # 'undo' import of foo easyblock + del sys.modules['easybuild.easyblocks.generic.binary'] + sys.path[:] = orig_local_sys_path + import easybuild.easyblocks + reload(easybuild.easyblocks) + import easybuild.easyblocks.generic + reload(easybuild.easyblocks.generic) + + pattern = "== easyblock binary.py included from comit %s" % test_commit self.assertEqual(stderr, '') self.assertIn(pattern, stdout) From 818e65a6e13981a3bd961adf2107d3ad5ad48a7e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 09:14:28 +0200 Subject: [PATCH 172/430] update copyright lines to 2024 --- easybuild/__init__.py | 2 +- easybuild/base/exceptions.py | 2 +- easybuild/base/fancylogger.py | 2 +- easybuild/base/generaloption.py | 2 +- easybuild/base/testing.py | 2 +- easybuild/framework/__init__.py | 2 +- easybuild/framework/easyblock.py | 2 +- easybuild/framework/easyconfig/__init__.py | 2 +- easybuild/framework/easyconfig/constants.py | 2 +- easybuild/framework/easyconfig/default.py | 2 +- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/format/__init__.py | 2 +- easybuild/framework/easyconfig/format/convert.py | 2 +- easybuild/framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/format/one.py | 2 +- easybuild/framework/easyconfig/format/pyheaderconfigobj.py | 2 +- easybuild/framework/easyconfig/format/two.py | 2 +- easybuild/framework/easyconfig/format/version.py | 2 +- easybuild/framework/easyconfig/format/yeb.py | 2 +- easybuild/framework/easyconfig/licenses.py | 2 +- easybuild/framework/easyconfig/parser.py | 2 +- easybuild/framework/easyconfig/style.py | 2 +- easybuild/framework/easyconfig/templates.py | 2 +- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/framework/easyconfig/types.py | 2 +- easybuild/framework/easystack.py | 2 +- easybuild/framework/extension.py | 2 +- easybuild/framework/extensioneasyblock.py | 2 +- easybuild/main.py | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- easybuild/scripts/clean_gists.py | 2 +- easybuild/scripts/fix_docs.py | 2 +- easybuild/scripts/mk_tmpl_easyblock_for.py | 2 +- easybuild/scripts/rpath_args.py | 2 +- easybuild/scripts/rpath_wrapper_template.sh.in | 2 +- easybuild/toolchains/__init__.py | 2 +- easybuild/toolchains/cgmpich.py | 2 +- easybuild/toolchains/cgmpolf.py | 2 +- easybuild/toolchains/cgmvapich2.py | 2 +- easybuild/toolchains/cgmvolf.py | 2 +- easybuild/toolchains/cgompi.py | 2 +- easybuild/toolchains/cgoolf.py | 2 +- easybuild/toolchains/clanggcc.py | 2 +- easybuild/toolchains/compiler/__init__.py | 2 +- easybuild/toolchains/compiler/clang.py | 2 +- easybuild/toolchains/compiler/craype.py | 2 +- easybuild/toolchains/compiler/cuda.py | 2 +- easybuild/toolchains/compiler/dummycompiler.py | 2 +- easybuild/toolchains/compiler/fujitsu.py | 2 +- easybuild/toolchains/compiler/gcc.py | 2 +- easybuild/toolchains/compiler/intel_compilers.py | 2 +- easybuild/toolchains/compiler/inteliccifort.py | 2 +- easybuild/toolchains/compiler/nvhpc.py | 2 +- easybuild/toolchains/compiler/pgi.py | 2 +- easybuild/toolchains/compiler/systemcompiler.py | 2 +- easybuild/toolchains/craycce.py | 2 +- easybuild/toolchains/craygnu.py | 2 +- easybuild/toolchains/crayintel.py | 2 +- easybuild/toolchains/craypgi.py | 2 +- easybuild/toolchains/dummy.py | 2 +- easybuild/toolchains/fcc.py | 2 +- easybuild/toolchains/ffmpi.py | 2 +- easybuild/toolchains/fft/__init__.py | 2 +- easybuild/toolchains/fft/fftw.py | 2 +- easybuild/toolchains/fft/fujitsufftw.py | 2 +- easybuild/toolchains/fft/intelfftw.py | 2 +- easybuild/toolchains/foss.py | 2 +- easybuild/toolchains/fosscuda.py | 2 +- easybuild/toolchains/fujitsu.py | 2 +- easybuild/toolchains/gcc.py | 2 +- easybuild/toolchains/gcccore.py | 2 +- easybuild/toolchains/gcccuda.py | 2 +- easybuild/toolchains/gfbf.py | 2 +- easybuild/toolchains/gimkl.py | 2 +- easybuild/toolchains/gimpi.py | 2 +- easybuild/toolchains/gimpic.py | 2 +- easybuild/toolchains/giolf.py | 2 +- easybuild/toolchains/giolfc.py | 2 +- easybuild/toolchains/gmacml.py | 2 +- easybuild/toolchains/gmkl.py | 2 +- easybuild/toolchains/gmklc.py | 2 +- easybuild/toolchains/gmpich.py | 2 +- easybuild/toolchains/gmpich2.py | 2 +- easybuild/toolchains/gmpit.py | 2 +- easybuild/toolchains/gmpolf.py | 2 +- easybuild/toolchains/gmvapich2.py | 2 +- easybuild/toolchains/gmvolf.py | 2 +- easybuild/toolchains/gnu.py | 2 +- easybuild/toolchains/goalf.py | 2 +- easybuild/toolchains/gobff.py | 2 +- easybuild/toolchains/goblf.py | 2 +- easybuild/toolchains/gofbf.py | 2 +- easybuild/toolchains/golf.py | 2 +- easybuild/toolchains/golfc.py | 2 +- easybuild/toolchains/gomkl.py | 2 +- easybuild/toolchains/gomklc.py | 2 +- easybuild/toolchains/gompi.py | 2 +- easybuild/toolchains/gompic.py | 2 +- easybuild/toolchains/goolf.py | 2 +- easybuild/toolchains/goolfc.py | 2 +- easybuild/toolchains/gpsmpi.py | 2 +- easybuild/toolchains/gpsolf.py | 2 +- easybuild/toolchains/gqacml.py | 2 +- easybuild/toolchains/gsmpi.py | 2 +- easybuild/toolchains/gsolf.py | 2 +- easybuild/toolchains/iccifort.py | 2 +- easybuild/toolchains/iccifortcuda.py | 2 +- easybuild/toolchains/ictce.py | 2 +- easybuild/toolchains/ifbf.py | 2 +- easybuild/toolchains/iibff.py | 2 +- easybuild/toolchains/iimkl.py | 2 +- easybuild/toolchains/iimklc.py | 2 +- easybuild/toolchains/iimpi.py | 2 +- easybuild/toolchains/iimpic.py | 2 +- easybuild/toolchains/iiqmpi.py | 2 +- easybuild/toolchains/impich.py | 2 +- easybuild/toolchains/impmkl.py | 2 +- easybuild/toolchains/intel-para.py | 2 +- easybuild/toolchains/intel.py | 2 +- easybuild/toolchains/intel_compilers.py | 2 +- easybuild/toolchains/intelcuda.py | 2 +- easybuild/toolchains/iofbf.py | 2 +- easybuild/toolchains/iomkl.py | 2 +- easybuild/toolchains/iomklc.py | 2 +- easybuild/toolchains/iompi.py | 2 +- easybuild/toolchains/iompic.py | 2 +- easybuild/toolchains/ipsmpi.py | 2 +- easybuild/toolchains/iqacml.py | 2 +- easybuild/toolchains/ismkl.py | 2 +- easybuild/toolchains/linalg/__init__.py | 2 +- easybuild/toolchains/linalg/acml.py | 2 +- easybuild/toolchains/linalg/atlas.py | 2 +- easybuild/toolchains/linalg/blacs.py | 2 +- easybuild/toolchains/linalg/blis.py | 2 +- easybuild/toolchains/linalg/flame.py | 2 +- easybuild/toolchains/linalg/flexiblas.py | 2 +- easybuild/toolchains/linalg/fujitsussl.py | 2 +- easybuild/toolchains/linalg/gotoblas.py | 2 +- easybuild/toolchains/linalg/intelmkl.py | 2 +- easybuild/toolchains/linalg/lapack.py | 2 +- easybuild/toolchains/linalg/libsci.py | 2 +- easybuild/toolchains/linalg/openblas.py | 2 +- easybuild/toolchains/linalg/scalapack.py | 2 +- easybuild/toolchains/mpi/__init__.py | 2 +- easybuild/toolchains/mpi/craympich.py | 2 +- easybuild/toolchains/mpi/fujitsumpi.py | 2 +- easybuild/toolchains/mpi/intelmpi.py | 2 +- easybuild/toolchains/mpi/mpich.py | 2 +- easybuild/toolchains/mpi/mpich2.py | 2 +- easybuild/toolchains/mpi/mpitrampoline.py | 2 +- easybuild/toolchains/mpi/mvapich2.py | 2 +- easybuild/toolchains/mpi/openmpi.py | 2 +- easybuild/toolchains/mpi/psmpi.py | 2 +- easybuild/toolchains/mpi/qlogicmpi.py | 2 +- easybuild/toolchains/mpi/spectrummpi.py | 2 +- easybuild/toolchains/nvhpc.py | 2 +- easybuild/toolchains/nvofbf.py | 2 +- easybuild/toolchains/nvompi.py | 2 +- easybuild/toolchains/nvompic.py | 4 ++-- easybuild/toolchains/nvpsmpi.py | 4 ++-- easybuild/toolchains/nvpsmpic.py | 4 ++-- easybuild/toolchains/pgi.py | 2 +- easybuild/toolchains/pmkl.py | 2 +- easybuild/toolchains/pomkl.py | 2 +- easybuild/toolchains/pompi.py | 2 +- easybuild/toolchains/system.py | 2 +- easybuild/tools/__init__.py | 2 +- easybuild/tools/asyncprocess.py | 2 +- easybuild/tools/build_details.py | 2 +- easybuild/tools/build_log.py | 2 +- easybuild/tools/config.py | 2 +- easybuild/tools/containers/__init__.py | 2 +- easybuild/tools/containers/apptainer.py | 2 +- easybuild/tools/containers/base.py | 2 +- easybuild/tools/containers/common.py | 2 +- easybuild/tools/containers/docker.py | 2 +- easybuild/tools/containers/singularity.py | 2 +- easybuild/tools/containers/utils.py | 2 +- easybuild/tools/convert.py | 2 +- easybuild/tools/docs.py | 2 +- easybuild/tools/environment.py | 2 +- easybuild/tools/filetools.py | 2 +- easybuild/tools/github.py | 2 +- easybuild/tools/hooks.py | 2 +- easybuild/tools/include.py | 2 +- easybuild/tools/jenkins.py | 2 +- easybuild/tools/job/backend.py | 2 +- easybuild/tools/job/gc3pie.py | 2 +- easybuild/tools/job/pbs_python.py | 2 +- easybuild/tools/job/slurm.py | 2 +- easybuild/tools/module_generator.py | 2 +- easybuild/tools/module_naming_scheme/__init__.py | 2 +- easybuild/tools/module_naming_scheme/categorized_mns.py | 2 +- easybuild/tools/module_naming_scheme/easybuild_mns.py | 2 +- easybuild/tools/module_naming_scheme/hierarchical_mns.py | 2 +- .../tools/module_naming_scheme/migrate_from_eb_to_hmns.py | 2 +- easybuild/tools/module_naming_scheme/mns.py | 2 +- easybuild/tools/module_naming_scheme/toolchain.py | 2 +- easybuild/tools/module_naming_scheme/utilities.py | 2 +- easybuild/tools/modules.py | 2 +- easybuild/tools/multidiff.py | 2 +- easybuild/tools/options.py | 2 +- easybuild/tools/output.py | 2 +- .../package_naming_scheme/easybuild_deb_friendly_pns.py | 2 +- .../tools/package/package_naming_scheme/easybuild_pns.py | 2 +- easybuild/tools/package/package_naming_scheme/pns.py | 2 +- easybuild/tools/package/utilities.py | 2 +- easybuild/tools/parallelbuild.py | 2 +- easybuild/tools/py2vs3/__init__.py | 2 +- easybuild/tools/py2vs3/py2.py | 2 +- easybuild/tools/py2vs3/py3.py | 2 +- easybuild/tools/repository/filerepo.py | 2 +- easybuild/tools/repository/gitrepo.py | 2 +- easybuild/tools/repository/hgrepo.py | 2 +- easybuild/tools/repository/repository.py | 2 +- easybuild/tools/repository/svnrepo.py | 2 +- easybuild/tools/robot.py | 2 +- easybuild/tools/run.py | 2 +- easybuild/tools/systemtools.py | 2 +- easybuild/tools/testing.py | 2 +- easybuild/tools/toolchain/__init__.py | 2 +- easybuild/tools/toolchain/compiler.py | 2 +- easybuild/tools/toolchain/constants.py | 2 +- easybuild/tools/toolchain/fft.py | 2 +- easybuild/tools/toolchain/linalg.py | 2 +- easybuild/tools/toolchain/mpi.py | 2 +- easybuild/tools/toolchain/options.py | 2 +- easybuild/tools/toolchain/toolchain.py | 2 +- easybuild/tools/toolchain/toolchainvariables.py | 2 +- easybuild/tools/toolchain/utilities.py | 2 +- easybuild/tools/toolchain/variables.py | 2 +- easybuild/tools/utilities.py | 2 +- easybuild/tools/variables.py | 2 +- easybuild/tools/version.py | 2 +- eb | 2 +- setup.py | 2 +- test/__init__.py | 2 +- test/framework/__init__.py | 2 +- test/framework/asyncprocess.py | 2 +- test/framework/build_log.py | 2 +- test/framework/config.py | 2 +- test/framework/containers.py | 2 +- test/framework/docs.py | 2 +- test/framework/easyblock.py | 2 +- test/framework/easyconfig.py | 2 +- test/framework/easyconfigformat.py | 2 +- test/framework/easyconfigparser.py | 2 +- test/framework/easyconfigversion.py | 2 +- test/framework/easystack.py | 2 +- test/framework/ebconfigobj.py | 2 +- test/framework/environment.py | 2 +- test/framework/filetools.py | 2 +- test/framework/format_convert.py | 2 +- test/framework/general.py | 2 +- test/framework/github.py | 2 +- test/framework/hooks.py | 2 +- test/framework/include.py | 2 +- test/framework/lib.py | 2 +- test/framework/license.py | 2 +- test/framework/module_generator.py | 2 +- test/framework/modules.py | 2 +- test/framework/modulestool.py | 2 +- test/framework/options.py | 2 +- test/framework/output.py | 2 +- test/framework/package.py | 2 +- test/framework/parallelbuild.py | 2 +- test/framework/repository.py | 2 +- test/framework/robot.py | 2 +- test/framework/run.py | 2 +- .../framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/fftw.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/foo.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/foofoo.py | 2 +- test/framework/sandbox/easybuild/easyblocks/g/gcc.py | 2 +- test/framework/sandbox/easybuild/easyblocks/generic/bar.py | 2 +- .../sandbox/easybuild/easyblocks/generic/configuremake.py | 2 +- .../sandbox/easybuild/easyblocks/generic/dummyextension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/generic/makecp.py | 2 +- .../sandbox/easybuild/easyblocks/generic/modulerc.py | 2 +- .../sandbox/easybuild/easyblocks/generic/pythonbundle.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toolchain.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/h/hpl.py | 2 +- test/framework/sandbox/easybuild/easyblocks/l/libtoy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/o/openblas.py | 2 +- test/framework/sandbox/easybuild/easyblocks/o/openmpi.py | 2 +- test/framework/sandbox/easybuild/easyblocks/s/scalapack.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toytoy.py | 2 +- test/framework/sandbox/easybuild/tools/__init__.py | 2 +- .../sandbox/easybuild/tools/module_naming_scheme/__init__.py | 2 +- .../tools/module_naming_scheme/broken_module_naming_scheme.py | 2 +- .../tools/module_naming_scheme/test_module_naming_scheme.py | 2 +- .../module_naming_scheme/test_module_naming_scheme_more.py | 2 +- test/framework/style.py | 2 +- test/framework/suite.py | 2 +- test/framework/systemtools.py | 2 +- test/framework/toolchain.py | 2 +- test/framework/toolchainvariables.py | 2 +- test/framework/toy_build.py | 2 +- test/framework/tweak.py | 2 +- test/framework/type_checking.py | 2 +- test/framework/utilities.py | 2 +- test/framework/utilities_test.py | 2 +- test/framework/variables.py | 2 +- test/framework/yeb.py | 2 +- 309 files changed, 312 insertions(+), 312 deletions(-) diff --git a/easybuild/__init__.py b/easybuild/__init__.py index 4763af8bc4..3637c9f395 100644 --- a/easybuild/__init__.py +++ b/easybuild/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/exceptions.py b/easybuild/base/exceptions.py index 3bde7cc6a1..eb004ad217 100644 --- a/easybuild/base/exceptions.py +++ b/easybuild/base/exceptions.py @@ -1,5 +1,5 @@ # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index 93582b0cf1..4efe6ee615 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index c5af9220d1..cfd1a37e1f 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index 01b64d84ef..4b3df7e563 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -1,5 +1,5 @@ # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/__init__.py b/easybuild/framework/__init__.py index 019e507d79..f03298abca 100644 --- a/easybuild/framework/__init__.py +++ b/easybuild/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 063b2c01cc..d2727d61ee 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py index 6d91c028a9..b19017e023 100644 --- a/easybuild/framework/easyconfig/__init__.py +++ b/easybuild/framework/easyconfig/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index 2e62580db4..4fe8c9d7b5 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dae27aca65..172995246e 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ff4ed3562c..6e5f6aae35 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/__init__.py b/easybuild/framework/easyconfig/format/__init__.py index b2c084e273..c2a81766ad 100644 --- a/easybuild/framework/easyconfig/format/__init__.py +++ b/easybuild/framework/easyconfig/format/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/convert.py b/easybuild/framework/easyconfig/format/convert.py index 726acd3e6c..82613d42e0 100644 --- a/easybuild/framework/easyconfig/format/convert.py +++ b/easybuild/framework/easyconfig/format/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index d503b6703a..3aeae419a2 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 680e4ee77b..868e218925 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 37e14b529e..f5889f5e2c 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/two.py b/easybuild/framework/easyconfig/format/two.py index cec3df3636..652b1c9b80 100644 --- a/easybuild/framework/easyconfig/format/two.py +++ b/easybuild/framework/easyconfig/format/two.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index fe3b1a2316..54d05de9f9 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index acfab3fb22..675d0f0a86 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/licenses.py b/easybuild/framework/easyconfig/licenses.py index e8eb1ee3bc..e6286d54e3 100644 --- a/easybuild/framework/easyconfig/licenses.py +++ b/easybuild/framework/easyconfig/licenses.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index fa01d1b297..d6521b17dc 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py index 0015b0c9b9..bad6ab5bda 100644 --- a/easybuild/framework/easyconfig/style.py +++ b/easybuild/framework/easyconfig/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index aed8db6af4..80d38e8bad 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 73393040c6..7d461c1461 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 7aaa69b69a..fb01f3a83e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index 622479f480..42be1b99e4 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 9296c18827..4a0ec15cdd 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -1,4 +1,4 @@ -# Copyright 2020-2023 Ghent University +# Copyright 2020-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index ed90f15e6d..56f242ffce 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 6ad116b20b..69c824fe7e 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). diff --git a/easybuild/main.py b/easybuild/main.py index 85501f57f0..41eefb4461 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index a74f4fa55e..ef8c281162 100755 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index cb2d6fed04..ec1213e4b3 100755 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2014 Ward Poelmans +# Copyright 2014-2024 Ward Poelmans # # https://github.com/easybuilders/easybuild # diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py index 388fb360d9..a14f18caf8 100755 --- a/easybuild/scripts/fix_docs.py +++ b/easybuild/scripts/fix_docs.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index c5c7c0110a..f6958572fa 100755 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 1bc87a1679..b477aa63d8 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 55c3388c5b..44a5aac43a 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -1,6 +1,6 @@ #!/bin/bash ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/__init__.py b/easybuild/toolchains/__init__.py index 309b1808cf..2f468f2812 100644 --- a/easybuild/toolchains/__init__.py +++ b/easybuild/toolchains/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py index ea04e20e75..9e25c6ec57 100644 --- a/easybuild/toolchains/cgmpich.py +++ b/easybuild/toolchains/cgmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmpolf.py b/easybuild/toolchains/cgmpolf.py index b335781374..5629ee5be3 100644 --- a/easybuild/toolchains/cgmpolf.py +++ b/easybuild/toolchains/cgmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py index e347cab483..76e306b12d 100644 --- a/easybuild/toolchains/cgmvapich2.py +++ b/easybuild/toolchains/cgmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvolf.py b/easybuild/toolchains/cgmvolf.py index 1b9e122fcd..e68aa90fe1 100644 --- a/easybuild/toolchains/cgmvolf.py +++ b/easybuild/toolchains/cgmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py index 559f91f4ad..9b9267bf78 100644 --- a/easybuild/toolchains/cgompi.py +++ b/easybuild/toolchains/cgompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgoolf.py b/easybuild/toolchains/cgoolf.py index c26e94b724..8e2ea8503e 100644 --- a/easybuild/toolchains/cgoolf.py +++ b/easybuild/toolchains/cgoolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/clanggcc.py b/easybuild/toolchains/clanggcc.py index 8537354ebe..c1e09e68ef 100644 --- a/easybuild/toolchains/clanggcc.py +++ b/easybuild/toolchains/clanggcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/__init__.py b/easybuild/toolchains/compiler/__init__.py index 5933777f43..9c1ba469c5 100644 --- a/easybuild/toolchains/compiler/__init__.py +++ b/easybuild/toolchains/compiler/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 8c93053932..8b7b16e8e3 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py index a0bec9f624..73b6a103e2 100644 --- a/easybuild/toolchains/compiler/craype.py +++ b/easybuild/toolchains/compiler/craype.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index c87f1b21d1..3fd3d705f1 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/dummycompiler.py b/easybuild/toolchains/compiler/dummycompiler.py index 767dcb66c5..d74c2e873e 100644 --- a/easybuild/toolchains/compiler/dummycompiler.py +++ b/easybuild/toolchains/compiler/dummycompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index e7a861c1b7..437ec34048 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index fd20846592..548ac41187 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py index ae97dfa87d..1a21b21943 100644 --- a/easybuild/toolchains/compiler/intel_compilers.py +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index c498f544d7..01015a0d83 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/nvhpc.py b/easybuild/toolchains/compiler/nvhpc.py index 549c0e9a70..2011137716 100644 --- a/easybuild/toolchains/compiler/nvhpc.py +++ b/easybuild/toolchains/compiler/nvhpc.py @@ -1,5 +1,5 @@ ## -# Copyright 2015 Bart Oldeman +# Copyright 2015-2024 Bart Oldeman # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/pgi.py b/easybuild/toolchains/compiler/pgi.py index 77f04fe813..0f55614926 100644 --- a/easybuild/toolchains/compiler/pgi.py +++ b/easybuild/toolchains/compiler/pgi.py @@ -1,5 +1,5 @@ ## -# Copyright 2015 Bart Oldeman +# Copyright 2015-2024 Bart Oldeman # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/systemcompiler.py b/easybuild/toolchains/compiler/systemcompiler.py index 505f002d57..d1fd06d4d9 100644 --- a/easybuild/toolchains/compiler/systemcompiler.py +++ b/easybuild/toolchains/compiler/systemcompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craycce.py b/easybuild/toolchains/craycce.py index cb9a4ff80e..13b92e55d2 100644 --- a/easybuild/toolchains/craycce.py +++ b/easybuild/toolchains/craycce.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craygnu.py b/easybuild/toolchains/craygnu.py index 4e42d32f94..8d198362bd 100644 --- a/easybuild/toolchains/craygnu.py +++ b/easybuild/toolchains/craygnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/crayintel.py b/easybuild/toolchains/crayintel.py index 76cde0a8bc..5997eb63d6 100644 --- a/easybuild/toolchains/crayintel.py +++ b/easybuild/toolchains/crayintel.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craypgi.py b/easybuild/toolchains/craypgi.py index 67e2bcec78..49004775a4 100644 --- a/easybuild/toolchains/craypgi.py +++ b/easybuild/toolchains/craypgi.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/dummy.py b/easybuild/toolchains/dummy.py index cb2a988bd0..f29f6736ae 100644 --- a/easybuild/toolchains/dummy.py +++ b/easybuild/toolchains/dummy.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fcc.py b/easybuild/toolchains/fcc.py index 905835e3cf..bc99335c48 100644 --- a/easybuild/toolchains/fcc.py +++ b/easybuild/toolchains/fcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ffmpi.py b/easybuild/toolchains/ffmpi.py index 34a56d229e..71a49f7913 100644 --- a/easybuild/toolchains/ffmpi.py +++ b/easybuild/toolchains/ffmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/__init__.py b/easybuild/toolchains/fft/__init__.py index 9915268575..afcf0211dd 100644 --- a/easybuild/toolchains/fft/__init__.py +++ b/easybuild/toolchains/fft/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index 63d39cb658..9cb4c4e851 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/fujitsufftw.py b/easybuild/toolchains/fft/fujitsufftw.py index 1e2ed28ec6..4de1769528 100644 --- a/easybuild/toolchains/fft/fujitsufftw.py +++ b/easybuild/toolchains/fft/fujitsufftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index ad8f08ffd8..7882278a91 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index d5e57956a0..031daa4c9c 100644 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fosscuda.py b/easybuild/toolchains/fosscuda.py index a35c570781..9465515d36 100644 --- a/easybuild/toolchains/fosscuda.py +++ b/easybuild/toolchains/fosscuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py index ed858a84c7..707255e03c 100644 --- a/easybuild/toolchains/fujitsu.py +++ b/easybuild/toolchains/fujitsu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py index e359b8aa01..71091e795c 100644 --- a/easybuild/toolchains/gcc.py +++ b/easybuild/toolchains/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py index 87eaf4d1b6..ce4703e89c 100644 --- a/easybuild/toolchains/gcccore.py +++ b/easybuild/toolchains/gcccore.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py index fcf0981bac..47ab250824 100644 --- a/easybuild/toolchains/gcccuda.py +++ b/easybuild/toolchains/gcccuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gfbf.py b/easybuild/toolchains/gfbf.py index 9a7f279e6d..d41f5b7d06 100644 --- a/easybuild/toolchains/gfbf.py +++ b/easybuild/toolchains/gfbf.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py index 8505cf5b87..1d7b9704f6 100644 --- a/easybuild/toolchains/gimkl.py +++ b/easybuild/toolchains/gimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py index 6ac9788749..76e6bd638a 100644 --- a/easybuild/toolchains/gimpi.py +++ b/easybuild/toolchains/gimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpic.py b/easybuild/toolchains/gimpic.py index 01a28c78ee..dde1d6c8f8 100644 --- a/easybuild/toolchains/gimpic.py +++ b/easybuild/toolchains/gimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolf.py b/easybuild/toolchains/giolf.py index 64b99f2a5c..121c81121b 100644 --- a/easybuild/toolchains/giolf.py +++ b/easybuild/toolchains/giolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolfc.py b/easybuild/toolchains/giolfc.py index 3ca3111542..df798626aa 100644 --- a/easybuild/toolchains/giolfc.py +++ b/easybuild/toolchains/giolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmacml.py b/easybuild/toolchains/gmacml.py index 5561f85286..ee95c3f7f3 100644 --- a/easybuild/toolchains/gmacml.py +++ b/easybuild/toolchains/gmacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmkl.py b/easybuild/toolchains/gmkl.py index a58a05c488..67e33e1e62 100644 --- a/easybuild/toolchains/gmkl.py +++ b/easybuild/toolchains/gmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py index b7ecd19115..bead896d5e 100644 --- a/easybuild/toolchains/gmklc.py +++ b/easybuild/toolchains/gmklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py index 941421e50e..66d2b29123 100644 --- a/easybuild/toolchains/gmpich.py +++ b/easybuild/toolchains/gmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich2.py b/easybuild/toolchains/gmpich2.py index bad4e0bbaf..f734d3cb56 100644 --- a/easybuild/toolchains/gmpich2.py +++ b/easybuild/toolchains/gmpich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpit.py b/easybuild/toolchains/gmpit.py index 395940a639..a39baf934d 100644 --- a/easybuild/toolchains/gmpit.py +++ b/easybuild/toolchains/gmpit.py @@ -1,5 +1,5 @@ ## -# Copyright 2022-2023 Ghent University +# Copyright 2022-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py index 2bca2b44dd..c5a8be97d9 100644 --- a/easybuild/toolchains/gmpolf.py +++ b/easybuild/toolchains/gmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gmvapich2.py b/easybuild/toolchains/gmvapich2.py index 46ebb4d489..26aa6ad73e 100644 --- a/easybuild/toolchains/gmvapich2.py +++ b/easybuild/toolchains/gmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py index 50c943cde6..613d8a6ed3 100644 --- a/easybuild/toolchains/gmvolf.py +++ b/easybuild/toolchains/gmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gnu.py b/easybuild/toolchains/gnu.py index a16eed38bd..84b06af535 100644 --- a/easybuild/toolchains/gnu.py +++ b/easybuild/toolchains/gnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goalf.py b/easybuild/toolchains/goalf.py index d833e3c1c5..db1280907b 100644 --- a/easybuild/toolchains/goalf.py +++ b/easybuild/toolchains/goalf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gobff.py b/easybuild/toolchains/gobff.py index 2cbb366dfe..9df26ff706 100644 --- a/easybuild/toolchains/gobff.py +++ b/easybuild/toolchains/gobff.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goblf.py b/easybuild/toolchains/goblf.py index 618d813149..63e7c0c0a4 100644 --- a/easybuild/toolchains/goblf.py +++ b/easybuild/toolchains/goblf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gofbf.py b/easybuild/toolchains/gofbf.py index 71269a57df..3128c87091 100644 --- a/easybuild/toolchains/gofbf.py +++ b/easybuild/toolchains/gofbf.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/golf.py b/easybuild/toolchains/golf.py index c318ec9fee..6d046413a8 100644 --- a/easybuild/toolchains/golf.py +++ b/easybuild/toolchains/golf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/golfc.py b/easybuild/toolchains/golfc.py index bdce5ddcec..3b99dce5c0 100644 --- a/easybuild/toolchains/golfc.py +++ b/easybuild/toolchains/golfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gomkl.py b/easybuild/toolchains/gomkl.py index 663bc32daa..ba93882a1b 100644 --- a/easybuild/toolchains/gomkl.py +++ b/easybuild/toolchains/gomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py index fa19da4b50..95620d8360 100644 --- a/easybuild/toolchains/gomklc.py +++ b/easybuild/toolchains/gomklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py index 35db8a8330..7af87123c9 100644 --- a/easybuild/toolchains/gompi.py +++ b/easybuild/toolchains/gompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py index a9f08dcf2a..0beb2b4d10 100644 --- a/easybuild/toolchains/gompic.py +++ b/easybuild/toolchains/gompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py index 6577cb31c4..099f7bc11a 100644 --- a/easybuild/toolchains/goolf.py +++ b/easybuild/toolchains/goolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py index 26cd18ec9b..76cf4db890 100644 --- a/easybuild/toolchains/goolfc.py +++ b/easybuild/toolchains/goolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py index c6a7fd04b8..529d17dc5e 100644 --- a/easybuild/toolchains/gpsmpi.py +++ b/easybuild/toolchains/gpsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py index 1414178019..e87f3a5de7 100644 --- a/easybuild/toolchains/gpsolf.py +++ b/easybuild/toolchains/gpsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gqacml.py b/easybuild/toolchains/gqacml.py index 5533bd1f3a..7456b4011e 100644 --- a/easybuild/toolchains/gqacml.py +++ b/easybuild/toolchains/gqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsmpi.py b/easybuild/toolchains/gsmpi.py index 865dbc813b..68f1a6d0f0 100644 --- a/easybuild/toolchains/gsmpi.py +++ b/easybuild/toolchains/gsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py index 5d30493bb1..cb917ca7d3 100644 --- a/easybuild/toolchains/gsolf.py +++ b/easybuild/toolchains/gsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py index d1b6804a93..38ec224a5b 100644 --- a/easybuild/toolchains/iccifort.py +++ b/easybuild/toolchains/iccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifortcuda.py b/easybuild/toolchains/iccifortcuda.py index 5f024dc8ce..149cfc8e39 100644 --- a/easybuild/toolchains/iccifortcuda.py +++ b/easybuild/toolchains/iccifortcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py index 265f249306..2055346b37 100644 --- a/easybuild/toolchains/ictce.py +++ b/easybuild/toolchains/ictce.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ifbf.py b/easybuild/toolchains/ifbf.py index 3507e1eab9..9f689279cb 100644 --- a/easybuild/toolchains/ifbf.py +++ b/easybuild/toolchains/ifbf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iibff.py b/easybuild/toolchains/iibff.py index 9fe2b2b0c8..07a6b1e567 100644 --- a/easybuild/toolchains/iibff.py +++ b/easybuild/toolchains/iibff.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimkl.py b/easybuild/toolchains/iimkl.py index 9d3d6a97f3..605445cc5c 100644 --- a/easybuild/toolchains/iimkl.py +++ b/easybuild/toolchains/iimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py index e9508c95ad..89325c2e67 100644 --- a/easybuild/toolchains/iimklc.py +++ b/easybuild/toolchains/iimklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index 00d9745a42..9337365ec6 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py index 53ca437150..45a8df3804 100644 --- a/easybuild/toolchains/iimpic.py +++ b/easybuild/toolchains/iimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iiqmpi.py b/easybuild/toolchains/iiqmpi.py index aee09bb658..5ff1466ff3 100644 --- a/easybuild/toolchains/iiqmpi.py +++ b/easybuild/toolchains/iiqmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py index 2b7fba07f9..7feda33805 100644 --- a/easybuild/toolchains/impich.py +++ b/easybuild/toolchains/impich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py index 1653b31d5c..952bff34ce 100644 --- a/easybuild/toolchains/impmkl.py +++ b/easybuild/toolchains/impmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py index a9d4ca0313..ddfee3aaae 100644 --- a/easybuild/toolchains/intel-para.py +++ b/easybuild/toolchains/intel-para.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py index 7dd625588f..659b05faf2 100644 --- a/easybuild/toolchains/intel.py +++ b/easybuild/toolchains/intel.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel_compilers.py b/easybuild/toolchains/intel_compilers.py index ab5de03386..27fdfbebfd 100644 --- a/easybuild/toolchains/intel_compilers.py +++ b/easybuild/toolchains/intel_compilers.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intelcuda.py b/easybuild/toolchains/intelcuda.py index 343715e43b..bdd59abaab 100644 --- a/easybuild/toolchains/intelcuda.py +++ b/easybuild/toolchains/intelcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iofbf.py b/easybuild/toolchains/iofbf.py index 3410ffaf9f..cd3313d600 100644 --- a/easybuild/toolchains/iofbf.py +++ b/easybuild/toolchains/iofbf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py index cefe83c3b1..033277919c 100644 --- a/easybuild/toolchains/iomkl.py +++ b/easybuild/toolchains/iomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomklc.py b/easybuild/toolchains/iomklc.py index 102fa7a311..f40cd78e16 100644 --- a/easybuild/toolchains/iomklc.py +++ b/easybuild/toolchains/iomklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py index 318db78481..b578c45892 100644 --- a/easybuild/toolchains/iompi.py +++ b/easybuild/toolchains/iompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompic.py b/easybuild/toolchains/iompic.py index 1bcb2eac71..5b675f5621 100644 --- a/easybuild/toolchains/iompic.py +++ b/easybuild/toolchains/iompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py index 1a381414c3..a8a772adcb 100644 --- a/easybuild/toolchains/ipsmpi.py +++ b/easybuild/toolchains/ipsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iqacml.py b/easybuild/toolchains/iqacml.py index fc63968583..e4c5476cd4 100644 --- a/easybuild/toolchains/iqacml.py +++ b/easybuild/toolchains/iqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py index 90ff100eeb..4ccdc87054 100644 --- a/easybuild/toolchains/ismkl.py +++ b/easybuild/toolchains/ismkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/__init__.py b/easybuild/toolchains/linalg/__init__.py index 152bae1e3d..90cdb068a8 100644 --- a/easybuild/toolchains/linalg/__init__.py +++ b/easybuild/toolchains/linalg/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index b9e25b311d..268e3b0cc2 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/atlas.py b/easybuild/toolchains/linalg/atlas.py index 4d53f5bd32..bc894bb108 100644 --- a/easybuild/toolchains/linalg/atlas.py +++ b/easybuild/toolchains/linalg/atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blacs.py b/easybuild/toolchains/linalg/blacs.py index a3963e9daa..201681c51f 100644 --- a/easybuild/toolchains/linalg/blacs.py +++ b/easybuild/toolchains/linalg/blacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py index fa607d0527..d89f3b8d83 100644 --- a/easybuild/toolchains/linalg/blis.py +++ b/easybuild/toolchains/linalg/blis.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index 78ac8f1ca4..5f6f5cfe87 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index c266aff248..0e88c1a9b2 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py index ecaff56683..dde792455b 100644 --- a/easybuild/toolchains/linalg/fujitsussl.py +++ b/easybuild/toolchains/linalg/fujitsussl.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/gotoblas.py b/easybuild/toolchains/linalg/gotoblas.py index 17d98413a8..e8a99a651a 100644 --- a/easybuild/toolchains/linalg/gotoblas.py +++ b/easybuild/toolchains/linalg/gotoblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 201355be46..8a32d684d9 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/lapack.py b/easybuild/toolchains/linalg/lapack.py index e9821a659d..e26c9e2961 100644 --- a/easybuild/toolchains/linalg/lapack.py +++ b/easybuild/toolchains/linalg/lapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py index 87c42116f5..07ea64ed82 100644 --- a/easybuild/toolchains/linalg/libsci.py +++ b/easybuild/toolchains/linalg/libsci.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py index e28d4f7dd2..18118d1f01 100644 --- a/easybuild/toolchains/linalg/openblas.py +++ b/easybuild/toolchains/linalg/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/scalapack.py b/easybuild/toolchains/linalg/scalapack.py index ef4d24ef30..980e3bfdab 100644 --- a/easybuild/toolchains/linalg/scalapack.py +++ b/easybuild/toolchains/linalg/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/__init__.py b/easybuild/toolchains/mpi/__init__.py index aea95e9051..c5abd0f595 100644 --- a/easybuild/toolchains/mpi/__init__.py +++ b/easybuild/toolchains/mpi/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py index 4c62c45519..14fc8cfcd9 100644 --- a/easybuild/toolchains/mpi/craympich.py +++ b/easybuild/toolchains/mpi/craympich.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py index f81e1d66f0..769ce94507 100644 --- a/easybuild/toolchains/mpi/fujitsumpi.py +++ b/easybuild/toolchains/mpi/fujitsumpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index 6b9f86430a..8bc67fe788 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py index b0c42e57d8..51c926173f 100644 --- a/easybuild/toolchains/mpi/mpich.py +++ b/easybuild/toolchains/mpi/mpich.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich2.py b/easybuild/toolchains/mpi/mpich2.py index bd85d30532..b6f1f6f082 100644 --- a/easybuild/toolchains/mpi/mpich2.py +++ b/easybuild/toolchains/mpi/mpich2.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py index 5af054661c..7ded1102e5 100644 --- a/easybuild/toolchains/mpi/mpitrampoline.py +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -1,5 +1,5 @@ ## -# Copyright 2022-2023 Ghent University +# Copyright 2022-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mvapich2.py b/easybuild/toolchains/mpi/mvapich2.py index afb2602f44..4f69c970f5 100644 --- a/easybuild/toolchains/mpi/mvapich2.py +++ b/easybuild/toolchains/mpi/mvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index 630262f48b..93531f3355 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/psmpi.py b/easybuild/toolchains/mpi/psmpi.py index cfbf720a74..07e8c9af78 100644 --- a/easybuild/toolchains/mpi/psmpi.py +++ b/easybuild/toolchains/mpi/psmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/qlogicmpi.py b/easybuild/toolchains/mpi/qlogicmpi.py index 26c0663756..b7355cd3e1 100644 --- a/easybuild/toolchains/mpi/qlogicmpi.py +++ b/easybuild/toolchains/mpi/qlogicmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/spectrummpi.py b/easybuild/toolchains/mpi/spectrummpi.py index e0d4b2f530..1fee1d51b7 100644 --- a/easybuild/toolchains/mpi/spectrummpi.py +++ b/easybuild/toolchains/mpi/spectrummpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvhpc.py b/easybuild/toolchains/nvhpc.py index aca69d386f..b4b871f84f 100644 --- a/easybuild/toolchains/nvhpc.py +++ b/easybuild/toolchains/nvhpc.py @@ -1,5 +1,5 @@ ## -# Copyright 2015 Bart Oldeman +# Copyright 2015-2024 Bart Oldeman # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/nvofbf.py b/easybuild/toolchains/nvofbf.py index 41e89e5a32..62d3cb4d37 100644 --- a/easybuild/toolchains/nvofbf.py +++ b/easybuild/toolchains/nvofbf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvompi.py b/easybuild/toolchains/nvompi.py index 5f2e25f03d..3feedaae35 100644 --- a/easybuild/toolchains/nvompi.py +++ b/easybuild/toolchains/nvompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py index e4ac0d6106..b9c5abc298 100644 --- a/easybuild/toolchains/nvompic.py +++ b/easybuild/toolchains/nvompic.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2023 Ghent University -# Copyright 2016-2023 Forschungszentrum Juelich +# Copyright 2016-2024 Ghent University +# Copyright 2016-2024 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvpsmpi.py b/easybuild/toolchains/nvpsmpi.py index 9e102c9d53..cbaee42bc0 100644 --- a/easybuild/toolchains/nvpsmpi.py +++ b/easybuild/toolchains/nvpsmpi.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2021 Ghent University -# Copyright 2016-2021 Forschungszentrum Juelich +# Copyright 2016-2024 Ghent University +# Copyright 2016-2024 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py index 604630ff8e..0b916c30a0 100644 --- a/easybuild/toolchains/nvpsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2023 Ghent University -# Copyright 2016-2023 Forschungszentrum Juelich +# Copyright 2016-2024 Ghent University +# Copyright 2016-2024 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pgi.py b/easybuild/toolchains/pgi.py index 2ee3dc3839..df2258bda4 100644 --- a/easybuild/toolchains/pgi.py +++ b/easybuild/toolchains/pgi.py @@ -1,5 +1,5 @@ ## -# Copyright 2015 Bart Oldeman +# Copyright 2015-2024 Bart Oldeman # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/pmkl.py b/easybuild/toolchains/pmkl.py index a4ad73d7cd..8b22d1d35f 100644 --- a/easybuild/toolchains/pmkl.py +++ b/easybuild/toolchains/pmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pomkl.py b/easybuild/toolchains/pomkl.py index ea85e2d440..f28f445296 100644 --- a/easybuild/toolchains/pomkl.py +++ b/easybuild/toolchains/pomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pompi.py b/easybuild/toolchains/pompi.py index f8a9e00039..9de2e52050 100644 --- a/easybuild/toolchains/pompi.py +++ b/easybuild/toolchains/pompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/system.py b/easybuild/toolchains/system.py index cfb46a93b2..8a8b17cfaa 100644 --- a/easybuild/toolchains/system.py +++ b/easybuild/toolchains/system.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index dee4fc0d12..8d34fc25c5 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py index b3a7d330ff..0a3c133fc4 100644 --- a/easybuild/tools/asyncprocess.py +++ b/easybuild/tools/asyncprocess.py @@ -1,6 +1,6 @@ ## # Copyright 2005 Josiah Carlson -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson. # and released under the GPL v2 on March 14, 2012 diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index e99ddb4af9..487e6372a3 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -1,4 +1,4 @@ -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 215b8d38aa..0bbff702a1 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ee8eb5fc53..b00ff1f8b3 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/__init__.py b/easybuild/tools/containers/__init__.py index 76cb37219a..215ba61c0c 100644 --- a/easybuild/tools/containers/__init__.py +++ b/easybuild/tools/containers/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py index 67a6db5bc3..4cb252dac4 100644 --- a/easybuild/tools/containers/apptainer.py +++ b/easybuild/tools/containers/apptainer.py @@ -1,4 +1,4 @@ -# Copyright 2022-2023 Ghent University +# Copyright 2022-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/base.py b/easybuild/tools/containers/base.py index 68a49610d3..faf3b0cc7a 100644 --- a/easybuild/tools/containers/base.py +++ b/easybuild/tools/containers/base.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/common.py b/easybuild/tools/containers/common.py index bcbdce9a9b..6824c16864 100644 --- a/easybuild/tools/containers/common.py +++ b/easybuild/tools/containers/common.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index aa37a90873..9061bba762 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index dc9cbb39a1..809fdc35d1 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -1,4 +1,4 @@ -# Copyright 2017-2023 Ghent University +# Copyright 2017-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py index 65be9dadf5..a575254335 100644 --- a/easybuild/tools/containers/utils.py +++ b/easybuild/tools/containers/utils.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py index f085590988..134566812d 100644 --- a/easybuild/tools/convert.py +++ b/easybuild/tools/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 13f2c11c27..d610bc6e87 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index d68755ff24..db6a488183 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index bc938f4210..18784e0cce 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 96ed4d6492..5ffc95c1fb 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index f454974edb..f018c40607 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2023 Ghent University +# Copyright 2017-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 3898facd4c..0171d74f10 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py index 3c00d3961c..50ee563083 100644 --- a/easybuild/tools/jenkins.py +++ b/easybuild/tools/jenkins.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py index c92ead6a58..1629ff3504 100644 --- a/easybuild/tools/job/backend.py +++ b/easybuild/tools/job/backend.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index 005cff2e90..f6b57e0f2c 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # Copyright 2015 S3IT, University of Zurich # # This file is part of EasyBuild, diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index a4a020988e..308067ef7d 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py index 1f2dea776d..2666cbd6af 100644 --- a/easybuild/tools/job/slurm.py +++ b/easybuild/tools/job/slurm.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2023 Ghent University +# Copyright 2018-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 45582bbe5c..7f9344f5d8 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/__init__.py b/easybuild/tools/module_naming_scheme/__init__.py index bea5d52dc5..e54f04c31e 100644 --- a/easybuild/tools/module_naming_scheme/__init__.py +++ b/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/categorized_mns.py b/easybuild/tools/module_naming_scheme/categorized_mns.py index 616baf19c2..cdafc97cbb 100644 --- a/easybuild/tools/module_naming_scheme/categorized_mns.py +++ b/easybuild/tools/module_naming_scheme/categorized_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/easybuild_mns.py b/easybuild/tools/module_naming_scheme/easybuild_mns.py index c12ae9a614..93ff663e77 100644 --- a/easybuild/tools/module_naming_scheme/easybuild_mns.py +++ b/easybuild/tools/module_naming_scheme/easybuild_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 9f2e026eb8..9332738b26 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py index 8d0c34297f..5603b7db47 100644 --- a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py +++ b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index 487dbac063..fd60dbbe90 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index 135d996ffb..b85032e6e6 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index e46609d22b..018259ea4d 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 8df257fa90..cd56888803 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/multidiff.py b/easybuild/tools/multidiff.py index 0f8c7f5bac..99ee949e56 100644 --- a/easybuild/tools/multidiff.py +++ b/easybuild/tools/multidiff.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 0ea04c7672..1ce8abee30 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 0dc5b3b362..b6da8a9245 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # # -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py index 9ae9b37419..033687fac1 100644 --- a/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py +++ b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- vim: set fileencoding=utf-8 ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py index a6543a7d53..7a53a86101 100644 --- a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py +++ b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/pns.py b/easybuild/tools/package/package_naming_scheme/pns.py index 081a94b073..8683ae8d37 100644 --- a/easybuild/tools/package/package_naming_scheme/pns.py +++ b/easybuild/tools/package/package_naming_scheme/pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 087084c687..43b7828850 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 6c94517c69..3ab31e1efc 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/__init__.py b/easybuild/tools/py2vs3/__init__.py index 5d9854b39f..5d45e11dc3 100644 --- a/easybuild/tools/py2vs3/__init__.py +++ b/easybuild/tools/py2vs3/__init__.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index 1fec4650b1..f619279060 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index 8d9145179d..62e7a93723 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py index c7d0cb366d..b18c3c6159 100644 --- a/easybuild/tools/repository/filerepo.py +++ b/easybuild/tools/repository/filerepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py index 97fc6c33da..feaf8cb2ad 100644 --- a/easybuild/tools/repository/gitrepo.py +++ b/easybuild/tools/repository/gitrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/hgrepo.py b/easybuild/tools/repository/hgrepo.py index 8cdf145b52..83a7890cc3 100644 --- a/easybuild/tools/repository/hgrepo.py +++ b/easybuild/tools/repository/hgrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index cbd33ab654..4948c461ac 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index 6c955c7fd0..79f593b1ee 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 68a4c9028b..b851b072ef 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 0685fb37f7..30f413be55 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 59b8728454..73eb657510 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index ffd8ce580b..bc7817721f 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/__init__.py b/easybuild/tools/toolchain/__init__.py index 876d16c44f..cf755a8e40 100644 --- a/easybuild/tools/toolchain/__init__.py +++ b/easybuild/tools/toolchain/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 5d9f025172..faf8f03d88 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index 5655f71961..643e8b4d21 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py index c0e1dedd1a..b7191d5a26 100644 --- a/easybuild/tools/toolchain/fft.py +++ b/easybuild/tools/toolchain/fft.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 907993571c..fd74615c4a 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 37c964907a..0ea2714490 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py index 06dcf9ee5d..391a2831e0 100644 --- a/easybuild/tools/toolchain/options.py +++ b/easybuild/tools/toolchain/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 4c30faf19a..409aac1f6c 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/toolchainvariables.py b/easybuild/tools/toolchain/toolchainvariables.py index 286b32aea3..dbb5686040 100644 --- a/easybuild/tools/toolchain/toolchainvariables.py +++ b/easybuild/tools/toolchain/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 56242b8fe4..8047778391 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index c201f8a6f8..482f989480 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index b879a4005c..8f6407e55d 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py index e5fa534c45..cef771f3d8 100644 --- a/easybuild/tools/variables.py +++ b/easybuild/tools/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index d296c6112d..006bb5543e 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/eb b/eb index d214d82b34..ad14ba61d0 100755 --- a/eb +++ b/eb @@ -1,6 +1,6 @@ #!/bin/bash ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/setup.py b/setup.py index 72ffd0e19a..82bae3aa2f 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/__init__.py b/test/__init__.py index 8793b2e4e1..20c9130dcc 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/__init__.py b/test/framework/__init__.py index c6a455e689..fae73ca9e6 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index f8db1927d8..c7eced6c48 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/build_log.py b/test/framework/build_log.py index a1792b998f..15f6984099 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/config.py b/test/framework/config.py index 4b42672cfc..55234d31b9 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/containers.py b/test/framework/containers.py index bd6dfde7f6..1d9abbf8e4 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2023 Ghent University +# Copyright 2018-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/docs.py b/test/framework/docs.py index 70280892e4..6b5b68ad17 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6c08f947d9..e6b54e0bc8 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 905cf6c15b..0112fd3c08 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py index e806a88378..7b84a2c867 100644 --- a/test/framework/easyconfigformat.py +++ b/test/framework/easyconfigformat.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index 9a17d8f0a5..fa943df938 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py index 5ef706a72a..4da54b450a 100644 --- a/test/framework/easyconfigversion.py +++ b/test/framework/easyconfigversion.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 91efcc94b3..a51059d280 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py index d4d81995d8..169fe988a2 100644 --- a/test/framework/ebconfigobj.py +++ b/test/framework/ebconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/environment.py b/test/framework/environment.py index 9a81e17486..5e90250d29 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8ee6b2c917..405ffc6b93 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py index 1bc3f764bb..e0add188b3 100644 --- a/test/framework/format_convert.py +++ b/test/framework/format_convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/general.py b/test/framework/general.py index 526fe07467..3837cecc87 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/github.py b/test/framework/github.py index f011c89b4f..33d29ea8fa 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 152d2352a4..5e810ac0f9 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2023 Ghent University +# Copyright 2017-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/include.py b/test/framework/include.py index b346f911ae..09a5925a2f 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/lib.py b/test/framework/lib.py index 2f4149314c..7930e00567 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2023 Ghent University +# Copyright 2018-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/license.py b/test/framework/license.py index 80c741908f..6dc79ac6a4 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index d2fbb6f642..456c18326e 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/modules.py b/test/framework/modules.py index 2a05395b7c..6fab651546 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index f43a91e3b3..8dd67a4fdb 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/options.py b/test/framework/options.py index a491936d78..a083a8495c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/output.py b/test/framework/output.py index d1673a55d1..4429fe29ee 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -1,5 +1,5 @@ # # -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/package.py b/test/framework/package.py index 23e2e37e9c..b839fb32bb 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 3ad85ac3b1..4e75672815 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/repository.py b/test/framework/repository.py index aa1cf402b7..720e0fa8e8 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/robot.py b/test/framework/robot.py index 4f226bae35..bf936dcb45 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/run.py b/test/framework/run.py index d529b41a3a..bab4391cf6 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1,6 +1,6 @@ # # # -*- coding: utf-8 -*- -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py index e27c0c66d0..7d2ef2d6da 100644 --- a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py +++ b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2020 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py index 194cead197..0871ce62e1 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foo.py b/test/framework/sandbox/easybuild/easyblocks/f/foo.py index 07e21608fa..b5a91d9503 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py index 3d64b1cf97..6415d6e6e1 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py index a85b2482fa..5ee5aae1c6 100644 --- a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py +++ b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py index d2faf66053..8ba475f700 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py index 4ad690fe5c..b82668434c 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py index d754117b39..3466ce28f2 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py index 6b258c87d6..dd89704638 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2020 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py index 33ab7db67f..72252a46f4 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py index 0321602f3f..9c01951adf 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2020 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py index fb81a51c96..5151221a13 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 9c700cf779..9335611f58 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py index 89fba75c30..db929d4d31 100644 --- a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py +++ b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py index 50b573649a..d3e55ed05d 100644 --- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py index bfb118cefa..e5dc093347 100644 --- a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py +++ b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py index 630d5413a7..b110b4f921 100644 --- a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py +++ b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py index d404f81648..3ded1247f4 100644 --- a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py +++ b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index c4614e3333..04186a37ef 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py index 3695744630..6445655707 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py index 0b6820adc9..09442d1a79 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2023 Ghent University +# Copyright 2020-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py index a1fc62c69b..a3ef746d53 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/__init__.py b/test/framework/sandbox/easybuild/tools/__init__.py index 1fbb2401b7..389c5ff367 100644 --- a/test/framework/sandbox/easybuild/tools/__init__.py +++ b/test/framework/sandbox/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py index 3a9383c79f..a519340298 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py index 4d33d3d1e7..9113d89045 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 5d8dec26c8..424fb55d43 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py index 3cbb5cb60d..a0e0429bfe 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/style.py b/test/framework/style.py index 298b165c24..bfdf4e026a 100644 --- a/test/framework/style.py +++ b/test/framework/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/suite.py b/test/framework/suite.py index 6a40e35754..3b8193469c 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -1,6 +1,6 @@ #!/usr/bin/python # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 45f51991ff..deece97702 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index aacde3baf7..91f4106a3e 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py index db90378d8b..e61bb9031c 100644 --- a/test/framework/toolchainvariables.py +++ b/test/framework/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 0349652b47..b699eb37ba 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/tweak.py b/test/framework/tweak.py index eda8046d52..24ec85b2c8 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index 83f1ef41ac..8b7edd3215 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 37a9b29318..3e60b45811 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index ba4766f302..a17d7beb02 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/variables.py b/test/framework/variables.py index 8331a6c2dd..cc804d6a65 100644 --- a/test/framework/variables.py +++ b/test/framework/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/yeb.py b/test/framework/yeb.py index 74a2a47875..346ffa057b 100644 --- a/test/framework/yeb.py +++ b/test/framework/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), From 717f55576974348aca7e96475aeff62b29a8c187 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 12:07:02 +0200 Subject: [PATCH 173/430] implement qa_wait_patterns option in run_shell_cmd --- easybuild/tools/run.py | 22 +++++++++++++++++----- test/framework/run.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index ba40b704ac..4376fb0220 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -252,9 +252,8 @@ def to_cmd_str(cmd): if not isinstance(qa_patterns, list) or any(not isinstance(x, tuple) or len(x) != 2 for x in qa_patterns): raise EasyBuildError("qa_patterns passed to run_shell_cmd should be a list of 2-tuples!") - # temporarily raise a NotImplementedError until all options are implemented - if qa_wait_patterns: - raise NotImplementedError + if qa_wait_patterns is None: + qa_wait_patterns = [] if work_dir is None: work_dir = os.getcwd() @@ -377,10 +376,10 @@ def to_cmd_str(cmd): if split_stderr: stderr += proc.stderr.read1(read_size) or b'' - # only consider answering questions if there's new output beyond additional whitespace if qa_patterns: + match_found = False for question, answers in qa_patterns: - + # allow extra whitespace at the end question += r'[\s\n]*$' regex = re.compile(question.encode()) if regex.search(stdout): @@ -397,8 +396,21 @@ def to_cmd_str(cmd): answer += '\n' os.write(proc.stdin.fileno(), answer.encode()) time_no_match = 0 + match_found = True break else: + # if no match was found among question patterns, + # take into account patterns for non-questions (qa_wait_patterns) + for pattern in qa_wait_patterns: + # allow extra whitespace at the end + pattern += r'[\s\n]*$' + regex = re.compile(pattern.encode()) + if regex.search(stdout): + time_no_match = 0 + match_found = True + break + + if not match_found: # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs if time_no_match > qa_timeout: diff --git a/test/framework/run.py b/test/framework/run.py index 6fb8d2b072..ba9a262925 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -771,6 +771,24 @@ def test_run_cmd_qa(self): self.assertTrue(out.startswith("question\nanswer\nfoo ")) self.assertTrue(out.endswith('bar')) + # test handling of output that is not actually a question + cmd = ';'.join([ + "echo not-a-question-but-a-statement", + "sleep 3", + "echo question", + "read x", + "echo $x", + ]) + qa = {'question': 'answer'} + + # fails because non-question is encountered + error_pattern = "Max nohits 1 reached: end of output not-a-question-but-a-statement" + self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd_qa, cmd, qa, maxhits=1, trace=False) + + (out, ec) = run_cmd_qa(cmd, qa, no_qa=["not-a-question-but-a-statement"], maxhits=1, trace=False) + self.assertEqual(out, "not-a-question-but-a-statement\nquestion\nanswer\n") + self.assertEqual(ec, 0) + def test_run_shell_cmd_qa(self): """Basic test for Q&A support in run_shell_cmd function.""" @@ -820,6 +838,27 @@ def test_run_shell_cmd_qa(self): with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1) + # test handling of output that is not actually a question + cmd = ';'.join([ + "echo not-a-question-but-a-statement", + "sleep 3", + "echo question", + "read x", + "echo $x", + ]) + qa = [('question', 'answer')] + + # fails because non-question is encountered + error_pattern = "No matching questions found for current command output, giving up after 1 seconds!" + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1, + hidden=True) + + qa_wait_patterns = ["not-a-question-but-a-statement"] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=qa_wait_patterns, qa_timeout=1) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "not-a-question-but-a-statement\nquestion\nanswer\n") + def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" From f020bf901f71c6a3bad2b65318c751deb9e71c51 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 12:08:11 +0200 Subject: [PATCH 174/430] fix excessively long line in test_run_shell_cmd_qa_answers --- test/framework/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index ba9a262925..0000a8efbd 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1047,7 +1047,8 @@ def test_run_shell_cmd_qa_answers(self): self.assertEqual(res.exit_code, 0) with self.mocked_stdout_stderr(): - self.assertErrorRegex(EasyBuildError, "Unknown type of answers encountered", run_shell_cmd, cmd, qa_patterns=[('question', 1)]) + self.assertErrorRegex(EasyBuildError, "Unknown type of answers encountered", run_shell_cmd, cmd, + qa_patterns=[('question', 1)]) # test cycling of answers cmd = cmd * 2 From 7d5c7ce51c9a4a852082df743dbbd4c90171a546 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 12:21:58 +0200 Subject: [PATCH 175/430] add support to run_shell_cmd for completing answer via pattern extracted from question --- easybuild/tools/run.py | 5 ++++- test/framework/run.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 4376fb0220..7fffe2ba6f 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -382,7 +382,8 @@ def to_cmd_str(cmd): # allow extra whitespace at the end question += r'[\s\n]*$' regex = re.compile(question.encode()) - if regex.search(stdout): + res = regex.search(stdout) + if res: # if answer is specified as a list, we take the first item as current answer, # and add it to the back of the list (so we cycle through answers) if isinstance(answers, list): @@ -393,6 +394,8 @@ def to_cmd_str(cmd): else: raise EasyBuildError(f"Unknown type of answers encountered: {answers}") + # answer may need to be completed via pattern extracted from question + answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()} answer += '\n' os.write(proc.stdin.fileno(), answer.encode()) time_no_match = 0 diff --git a/test/framework/run.py b/test/framework/run.py index 0000a8efbd..6ec6ebfb10 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -838,6 +838,18 @@ def test_run_shell_cmd_qa(self): with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1) + # check using answer that is completed via pattern extracted from question + cmd = ';'.join([ + "echo 'and the magic number is: 42'", + "read magic_number", + "echo $magic_number", + ]) + qa = [("and the magic number is: (?P[0-9]+)", "%(nr)s")] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "and the magic number is: 42\n42\n") + # test handling of output that is not actually a question cmd = ';'.join([ "echo not-a-question-but-a-statement", From 110f41c103f3e11fe7e031f48d889c021cbe9ea4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 15:41:52 +0200 Subject: [PATCH 176/430] fix comment in test_fetch_files_from_commit (not testing with short commit ID) Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- test/framework/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/github.py b/test/framework/github.py index 4bcf9b8987..e2bc20daf7 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -540,7 +540,7 @@ def test_github_fetch_files_from_pr_cache(self): def test_fetch_files_from_commit(self): """Test fetch_files_from_commit function.""" - # easyconfigs commit to add EasyBuild-4.8.2.eb (also test short commit, should work) + # easyconfigs commit to add EasyBuild-4.8.2.eb test_commit = '7c83a553950c233943c7b0189762f8c05cfea852' # without specifying any files/repo, default is to use easybuilders/easybuilld-easyconfigs From b9f6f51e87433694bf76177ccac577eea14604fe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 15:42:16 +0200 Subject: [PATCH 177/430] mention 'commit' option in docstring for download_repo --- easybuild/tools/github.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index eda237fca1..bd80c3c0e8 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -361,6 +361,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun Download entire GitHub repo as a tar.gz archive, and extract it into specified path. :param repo: repo to download :param branch: branch to download + :param commit: commit to download :param account: GitHub account to download repo from :param path: path to extract to :param github_user: name of GitHub user to use From 5f169a9f7845cef045a2365f9a25cbd452cb40fe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 15:44:01 +0200 Subject: [PATCH 178/430] GitHub token is not required in test_github_det_easyconfig_paths_from_commit --- test/framework/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/robot.py b/test/framework/robot.py index ccdf9a2b1e..680d2a70bc 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -715,6 +715,7 @@ def test_search_paths(self): def test_github_det_easyconfig_paths_from_commit(self): """Test det_easyconfig_paths function in combination with --from-commit.""" + # note: --from-commit does not involve using GitHub API, so no GitHub token required test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -730,7 +731,6 @@ def test_github_det_easyconfig_paths_from_commit(self): '--robot', '--robot=%s' % test_ecs_path, '--unittest-file=%s' % self.logfile, - '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user '--tmpdir=%s' % self.test_prefix, ] From 7145ca9676404eba51cef4e815b7c5745209f569 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 16:40:25 +0200 Subject: [PATCH 179/430] improve logging in Q&A part of run_shell_cmd --- easybuild/tools/run.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 7fffe2ba6f..26f2deed5c 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -286,12 +286,14 @@ def to_cmd_str(cmd): else: cmd_out_fp, cmd_err_fp = None, None + interactive = bool(qa_patterns) + interactive_msg = 'interactive ' if interactive else '' + # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled) if not in_dry_run and build_option('extended_dry_run'): if not hidden or verbose_dry_run: silent = build_option('silent') - interactive = 'interactive ' if qa_patterns else '' - msg = f" running {interactive}shell command \"{cmd_str}\"\n" + msg = f" running {interactive_msg}shell command \"{cmd_str}\"\n" msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) @@ -300,8 +302,7 @@ def to_cmd_str(cmd): start_time = datetime.now() if not hidden: - _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, - interactive=bool(qa_patterns)) + _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, interactive=interactive) if stream_output: print_msg(f"(streaming) output for command '{cmd_str}':") @@ -319,7 +320,7 @@ def to_cmd_str(cmd): if with_hooks: hooks = load_hooks(build_option('hooks')) kwargs = { - 'interactive': bool(qa_patterns), + 'interactive': interactive, 'work_dir': work_dir, } hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=kwargs) @@ -330,7 +331,7 @@ def to_cmd_str(cmd): stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT - log_msg = f"Running shell command '{cmd_str}' in {work_dir}" + log_msg = f"Running {interactive_msg}shell command '{cmd_str}' in {work_dir}" if thread_id: log_msg += f" (via thread with ID {thread_id})" _log.info(log_msg) @@ -384,6 +385,7 @@ def to_cmd_str(cmd): regex = re.compile(question.encode()) res = regex.search(stdout) if res: + _log.debug(f"Found match for question pattern '{question}' at end of: {stdout}") # if answer is specified as a list, we take the first item as current answer, # and add it to the back of the list (so we cycle through answers) if isinstance(answers, list): @@ -395,13 +397,16 @@ def to_cmd_str(cmd): raise EasyBuildError(f"Unknown type of answers encountered: {answers}") # answer may need to be completed via pattern extracted from question + _log.debug(f"Raw answer for question pattern '{question}': {answer}") answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()} answer += '\n' + _log.info(f"Found match for question pattern '{question}', replying with: {answer}") os.write(proc.stdin.fileno(), answer.encode()) time_no_match = 0 match_found = True break else: + _log.info("No match found for question patterns, considering question wait patterns") # if no match was found among question patterns, # take into account patterns for non-questions (qa_wait_patterns) for pattern in qa_wait_patterns: @@ -409,17 +414,24 @@ def to_cmd_str(cmd): pattern += r'[\s\n]*$' regex = re.compile(pattern.encode()) if regex.search(stdout): + _log.info(f"Found match for question wait pattern '{pattern}'") + _log.debug(f"Found match for question wait pattern '{pattern}' at end of: {stdout}") time_no_match = 0 match_found = True break + else: + _log.info("No match found for question wait patterns") if not match_found: + _log.debug(f"No match found in question or wait patterns at end of current output: {stdout}") # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs if time_no_match > qa_timeout: error_msg = "No matching questions found for current command output, " error_msg += f"giving up after {qa_timeout} seconds!" raise EasyBuildError(error_msg) + else: + _log.debug(f"{time_no_match} seconds without match in output of interactive shell command") time.sleep(check_interval_secs) @@ -467,7 +479,7 @@ def to_cmd_str(cmd): if with_hooks: run_hook_kwargs = { 'exit_code': res.exit_code, - 'interactive': bool(qa_patterns), + 'interactive': interactive, 'output': res.output, 'stderr': res.stderr, 'work_dir': res.work_dir, From f3e5a650d7dacd52e72c052dbe963b2483280b00 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 17:09:56 +0200 Subject: [PATCH 180/430] only show last 1000 characters in debug log message when answering questions for interactive commands in run_shell_cmd function --- easybuild/tools/run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 26f2deed5c..b63852b87c 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -385,7 +385,7 @@ def to_cmd_str(cmd): regex = re.compile(question.encode()) res = regex.search(stdout) if res: - _log.debug(f"Found match for question pattern '{question}' at end of: {stdout}") + _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout[:1000]}") # if answer is specified as a list, we take the first item as current answer, # and add it to the back of the list (so we cycle through answers) if isinstance(answers, list): @@ -414,8 +414,8 @@ def to_cmd_str(cmd): pattern += r'[\s\n]*$' regex = re.compile(pattern.encode()) if regex.search(stdout): - _log.info(f"Found match for question wait pattern '{pattern}'") - _log.debug(f"Found match for question wait pattern '{pattern}' at end of: {stdout}") + _log.info(f"Found match for wait pattern '{pattern}'") + _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout[:1000]}") time_no_match = 0 match_found = True break @@ -423,7 +423,7 @@ def to_cmd_str(cmd): _log.info("No match found for question wait patterns") if not match_found: - _log.debug(f"No match found in question or wait patterns at end of current output: {stdout}") + _log.debug(f"No match found in question/wait patterns at end of stdout: {stdout[:1000]}") # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs if time_no_match > qa_timeout: From ca21ff655675c0a28bbbf41a47d515ba93be4c4b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 18:13:45 +0200 Subject: [PATCH 181/430] only check for questions to be answered every 0.1s --- easybuild/tools/run.py | 4 ++-- test/framework/run.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index b63852b87c..bb26dc9482 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -360,7 +360,7 @@ def to_cmd_str(cmd): exit_code = None stdout, stderr = b'', b'' - check_interval_secs = 0.001 + check_interval_secs = 0.1 time_no_match = 0 # collect output piece-wise, while checking for questions to answer (if qa_patterns is provided) @@ -431,7 +431,7 @@ def to_cmd_str(cmd): error_msg += f"giving up after {qa_timeout} seconds!" raise EasyBuildError(error_msg) else: - _log.debug(f"{time_no_match} seconds without match in output of interactive shell command") + _log.debug(f"{time_no_match:0.1f} seconds without match in output of interactive shell command") time.sleep(check_interval_secs) diff --git a/test/framework/run.py b/test/framework/run.py index 6ec6ebfb10..6e64764805 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -907,8 +907,8 @@ def test_run_shell_cmd_qa_buffering(self): """Test whether run_shell_cmd uses unbuffered output when running interactive commands.""" # command that generates a lot of output before waiting for input - # note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000! - cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; ' + # note: bug being fixed can be reproduced reliably using 100, but not with too high values like 100000! + cmd = 'for x in $(seq 100); do echo "This is a number you can pick: $x"; done; ' cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"' with self.mocked_stdout_stderr(): res = run_shell_cmd(cmd, qa_patterns=[('Pick a number: ', '42')], qa_timeout=10) From 2abb706a839e49c022a2b621f572d6fae2e686eb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 18:26:23 +0200 Subject: [PATCH 182/430] flesh out code to try and answer question raised by interactive shell command from run_shell_cmd into private helper function _answer_question --- easybuild/tools/run.py | 104 +++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 45 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index bb26dc9482..24e860054b 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -202,6 +202,63 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): return ''.join([c for c in cmd if c in allowed_chars]) +def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): + """ + Private helper function to try and answer questions raised in interactive shell commands. + """ + match_found = False + + for question, answers in qa_patterns: + # allow extra whitespace at the end + question += r'[\s\n]*$' + regex = re.compile(question.encode()) + res = regex.search(stdout) + if res: + _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout[:1000]}") + # if answer is specified as a list, we take the first item as current answer, + # and add it to the back of the list (so we cycle through answers) + if isinstance(answers, list): + answer = answers.pop(0) + answers.append(answer) + elif isinstance(answers, str): + answer = answers + else: + raise EasyBuildError(f"Unknown type of answers encountered: {answers}") + + # answer may need to be completed via pattern extracted from question + _log.debug(f"Raw answer for question pattern '{question}': {answer}") + answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()} + answer += '\n' + _log.info(f"Found match for question pattern '{question}', replying with: {answer}") + + try: + os.write(proc.stdin.fileno(), answer.encode()) + except OSError as err: + raise EasyBuildError("Failed to answer question raised by interactive command: %s", err) + + time_no_match = 0 + match_found = True + break + else: + _log.info("No match found for question patterns, considering question wait patterns") + # if no match was found among question patterns, + # take into account patterns for non-questions (qa_wait_patterns) + for pattern in qa_wait_patterns: + # allow extra whitespace at the end + pattern += r'[\s\n]*$' + regex = re.compile(pattern.encode()) + if regex.search(stdout): + _log.info(f"Found match for wait pattern '{pattern}'") + _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout[:1000]}") + time_no_match = 0 + match_found = True + break + else: + _log.info("No match found for question wait patterns") + + return match_found + + @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, @@ -378,51 +435,7 @@ def to_cmd_str(cmd): stderr += proc.stderr.read1(read_size) or b'' if qa_patterns: - match_found = False - for question, answers in qa_patterns: - # allow extra whitespace at the end - question += r'[\s\n]*$' - regex = re.compile(question.encode()) - res = regex.search(stdout) - if res: - _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout[:1000]}") - # if answer is specified as a list, we take the first item as current answer, - # and add it to the back of the list (so we cycle through answers) - if isinstance(answers, list): - answer = answers.pop(0) - answers.append(answer) - elif isinstance(answers, str): - answer = answers - else: - raise EasyBuildError(f"Unknown type of answers encountered: {answers}") - - # answer may need to be completed via pattern extracted from question - _log.debug(f"Raw answer for question pattern '{question}': {answer}") - answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()} - answer += '\n' - _log.info(f"Found match for question pattern '{question}', replying with: {answer}") - os.write(proc.stdin.fileno(), answer.encode()) - time_no_match = 0 - match_found = True - break - else: - _log.info("No match found for question patterns, considering question wait patterns") - # if no match was found among question patterns, - # take into account patterns for non-questions (qa_wait_patterns) - for pattern in qa_wait_patterns: - # allow extra whitespace at the end - pattern += r'[\s\n]*$' - regex = re.compile(pattern.encode()) - if regex.search(stdout): - _log.info(f"Found match for wait pattern '{pattern}'") - _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout[:1000]}") - time_no_match = 0 - match_found = True - break - else: - _log.info("No match found for question wait patterns") - - if not match_found: + if not _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): _log.debug(f"No match found in question/wait patterns at end of stdout: {stdout[:1000]}") # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs @@ -437,6 +450,7 @@ def to_cmd_str(cmd): exit_code = proc.poll() + # collect last bit of output once processed has exited stdout += proc.stdout.read() if split_stderr: stderr += proc.stderr.read() From 23727d605563ce760a82ea4ba1fc9bc2b7306bad Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 18:27:38 +0200 Subject: [PATCH 183/430] don't reset time_no_match in _answer_question, do it in run_shell_cmd --- easybuild/tools/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 24e860054b..c1d13d01d0 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -236,7 +236,6 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): except OSError as err: raise EasyBuildError("Failed to answer question raised by interactive command: %s", err) - time_no_match = 0 match_found = True break else: @@ -250,7 +249,6 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): if regex.search(stdout): _log.info(f"Found match for wait pattern '{pattern}'") _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout[:1000]}") - time_no_match = 0 match_found = True break else: @@ -435,7 +433,9 @@ def to_cmd_str(cmd): stderr += proc.stderr.read1(read_size) or b'' if qa_patterns: - if not _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): + if _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): + time_no_match = 0 + else: _log.debug(f"No match found in question/wait patterns at end of stdout: {stdout[:1000]}") # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs From 6968f87da75b1e3b54c6bb6ee39e21d22f94f192 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 21:03:52 +0200 Subject: [PATCH 184/430] raise error when --from-commit and --from-pr or --include-easyblocks-from-commit and --include-easyblocks-from-pr are combined --- easybuild/main.py | 8 +++++++- easybuild/tools/options.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 4ca7bcbb95..8ffb185061 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -765,8 +765,14 @@ def prepare_main(args=None, logfile=None, testing=None): if __name__ == "__main__": - init_session_state, eb_go, cfg_settings = prepare_main() + # take into account that EasyBuildError may be raised when parsing the EasyBuild configuration + try: + init_session_state, eb_go, cfg_settings = prepare_main() + except EasyBuildError as err: + print_error(err.msg) + hooks = load_hooks(eb_go.options.hooks) + try: main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) except EasyBuildError as err: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 91bede9553..9eb6d915e5 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1512,6 +1512,19 @@ def parse_options(args=None, with_include=True): return eb_go +def check_options(options): + """ + Check configuration options, some combinations are not allowed. + """ + if options.from_commit and options.from_pr: + raise EasyBuildError("--from-commit and --from-pr should not be used together, pick one") + + if options.include_easyblocks_from_commit and options.include_easyblocks_from_pr: + error_msg = "--include-easyblocks-from-commit and --include-easyblocks-from-pr " + error_msg += "should not be used together, pick one" + raise EasyBuildError(error_msg) + + def check_root_usage(allow_use_as_root=False): """ Check whether we are running as root, and act accordingly @@ -1613,6 +1626,8 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r eb_go = parse_options(args=args) options = eb_go.options + check_options(options) + # tmpdir is set by option parser via set_tmpdir function tmpdir = tempfile.gettempdir() From 52648c09a33c17885ee0fb73847dbe9ef2aadb2a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 21:16:19 +0200 Subject: [PATCH 185/430] fix indentation in handle_include_easyblocks_from that breaks --include-easyblocks-from-pr if --include-easyblocks is not used (cfr. failing test_github_xxx_include_easyblocks_from_pr) --- easybuild/tools/options.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9eb6d915e5..a568c6263e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1574,8 +1574,7 @@ def check_included_multiple(included_easyblocks_from, source): if options.include_easyblocks: check_included_multiple(included_from_pr, "PR #%s" % easyblock_pr) - - included_easyblocks |= included_from_pr + included_easyblocks |= included_from_pr for easyblock in included_from_pr: print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log) From 640142adcefc164ac91ef51f2072cf59da1ceda8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 21:37:01 +0200 Subject: [PATCH 186/430] add comment to clarify why there's no early stop when --terse is combined with --missing-modules --- easybuild/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index 826880aad1..caaa4695e3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -684,6 +684,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr options.list_prs, options.merge_pr, options.review_pr, + # --missing-modules is processed by process_eb_args, + # so we can't exit just yet here if it's used in combination with --terse options.terse and not options.missing_modules, search_query, ] From b65692dc6a7a992eecdea786423682a72e016247 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 4 Apr 2024 10:18:43 +0800 Subject: [PATCH 187/430] prepare release notes for EasyBuild v4.9.1 + bump version to 4.9.1 --- RELEASE_NOTES | 35 +++++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 94b66db903..1d13b3a121 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,41 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.9.1 (5 April 2024) +--------------------- + +update/bugfix release + +- various enhancements, including: + - make `is_rpath_wrapper` faster by only checking file contents if file is not located in subdirectory of RPATH wrapper subdirectory (#4406) + - add terse support to `--missing-modules` (#4407) + - adapt version pattern for EnvironmentModules to allow using development version (#4416) + - use `--all` option with EnvironmentModules v4.6+ to get available hidden modules (#4417) + - add support for appending to path environment variables via `modextrapaths_append` (and add corresponding `allow_append_abs_path`) (#4436) + - improve output produced by `--check-github` (#4437) + - add script for updating local git repos with develop branch (#4438) + - show error when multiple PR options are passed (#4440) + - improve `findPythonDeps` script to recognize non-canonical package names (#4445) + - add support for `--from-commit` and `--include-easyblocks-from-commit` (#4468) + - improve logging & handling of (empty) `--optarch` values (#4481) + - add `--short` option to findUpdatedEcs (#4488) + - add generic GCC and Clang compiler flags for RISC-V (#4489) +- various bug fixes, including: + - clean up log file of EasyBlock instance in `check_sha256_checksums` (#4452) + - fix description of `backup-modules` configuration option (#4456) + - replace `'` with `"` for `printf` in CI workflow for running test suite to have bash replace a variable (#4461) + - use `cp -dR` instead of `cp -a` for shell script extraction (#4465) + - fix link to documentation in `close_pr` message (#4466) + - fix `test_github_merge_pr` by using more recent easyconfigs PR (#4470) + - add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration (#4472) +- other changes: + - clean up & speed up environment checks (#4409) + - use more performant and concise dict construction by using dict comprehensions (#4410) + - remove superflous string formatting (#4411) + - clean up uses of `getattr` and `hasattr` (#4412) + - update copyright lines to 2024 (#4494) + + v4.9.0 (30 December 2023) ------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 006bb5543e..0e82ff2ce9 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.1.dev0') +VERSION = LooseVersion('4.9.1') UNKNOWN = 'UNKNOWN' From cd1a860e5453c81bf4b25b7c9af40365d2d710bb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Apr 2024 17:23:47 +0200 Subject: [PATCH 188/430] minor tweaks to 4.9.1 release notes --- RELEASE_NOTES | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 1d13b3a121..07c9756e67 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -14,20 +14,20 @@ update/bugfix release - add terse support to `--missing-modules` (#4407) - adapt version pattern for EnvironmentModules to allow using development version (#4416) - use `--all` option with EnvironmentModules v4.6+ to get available hidden modules (#4417) - - add support for appending to path environment variables via `modextrapaths_append` (and add corresponding `allow_append_abs_path`) (#4436) + - add support for appending to path environment variables via `modextrapaths_append` + add corresponding `allow_append_abs_path` (#4436) - improve output produced by `--check-github` (#4437) - - add script for updating local git repos with develop branch (#4438) + - add script for updating local git repos with `develop` branch (#4438) - show error when multiple PR options are passed (#4440) - improve `findPythonDeps` script to recognize non-canonical package names (#4445) - add support for `--from-commit` and `--include-easyblocks-from-commit` (#4468) - improve logging & handling of (empty) `--optarch` values (#4481) - - add `--short` option to findUpdatedEcs (#4488) + - add `--short` option to `findUpdatedEcs` script (#4488) - add generic GCC and Clang compiler flags for RISC-V (#4489) - various bug fixes, including: - - clean up log file of EasyBlock instance in `check_sha256_checksums` (#4452) + - clean up log file of `EasyBlock` instance in `check_sha256_checksums` (#4452) - fix description of `backup-modules` configuration option (#4456) - replace `'` with `"` for `printf` in CI workflow for running test suite to have bash replace a variable (#4461) - - use `cp -dR` instead of `cp -a` for shell script extraction (#4465) + - use `cp -dR` instead of `cp -a` for shell script "extraction" (#4465) - fix link to documentation in `close_pr` message (#4466) - fix `test_github_merge_pr` by using more recent easyconfigs PR (#4470) - add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration (#4472) From ee73024d9c89ce4d528b56b1c29a9ed192daa03b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Apr 2024 20:21:21 +0200 Subject: [PATCH 189/430] bump version to 4.9.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 0e82ff2ce9..c01c8fc5c3 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.1') +VERSION = LooseVersion('4.9.2.dev0') UNKNOWN = 'UNKNOWN' From 204f3287b10974ba875337152ff0985fd3ae0083 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Apr 2024 10:19:47 +0200 Subject: [PATCH 190/430] tackle Simon's suggestions for run_shell_cmd --- easybuild/tools/run.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c1d13d01d0..3253a372b1 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -223,7 +223,7 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): elif isinstance(answers, str): answer = answers else: - raise EasyBuildError(f"Unknown type of answers encountered: {answers}") + raise EasyBuildError(f"Unknown type of answers encountered for question ({question}): {answers}") # answer may need to be completed via pattern extracted from question _log.debug(f"Raw answer for question pattern '{question}': {answer}") @@ -280,8 +280,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param task_id: task ID for specified shell command (included in return value) :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers - :param qa_wait_patterns: list of 2-tuples with patterns for non-questions - and number of iterations to allow these patterns to match with end out command output + :param qa_wait_patterns: list of strings with patterns for non-questions :param qa_timeout: amount of seconds to wait until more output is produced when there is no matching question :return: Named tuple with: From ce9a555a2f97737bfa1d0c263059de9eb17b823b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Apr 2024 13:13:00 +0200 Subject: [PATCH 191/430] disable trace for download_file used in fetch_files_from_commit --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 3bcc99b83c..d1b93c3ad7 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -637,7 +637,7 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff') diff_fn = os.path.basename(diff_url) diff_filepath = os.path.join(path, diff_fn) - if download_file(diff_fn, diff_url, diff_filepath, forced=True): + if download_file(diff_fn, diff_url, diff_filepath, forced=True, trace=False): diff_txt = read_file(diff_filepath) _log.debug("Diff for commit %s:\n%s", commit, diff_txt) From 965b3004969735671f4cfd04667b7999eb5fd05b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Apr 2024 14:06:26 +0200 Subject: [PATCH 192/430] tweak test_run_shell_cmd_qa_buffering to trigger buffering problem for Q&A shell command run via run_shell_cmd --- test/framework/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index aec941c552..f1ee6955ec 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -907,8 +907,8 @@ def test_run_shell_cmd_qa_buffering(self): """Test whether run_shell_cmd uses unbuffered output when running interactive commands.""" # command that generates a lot of output before waiting for input - # note: bug being fixed can be reproduced reliably using 100, but not with too high values like 100000! - cmd = 'for x in $(seq 100); do echo "This is a number you can pick: $x"; done; ' + # note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000! + cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; ' cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"' with self.mocked_stdout_stderr(): res = run_shell_cmd(cmd, qa_patterns=[('Pick a number: ', '42')], qa_timeout=10) From f899e2271ea0384c43b2cc1846a654c604bac5f0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Apr 2024 14:07:30 +0200 Subject: [PATCH 193/430] fix getting all available output for interactive commands in run_shell_cmd --- easybuild/tools/run.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 7f06d2ce06..7fa8806a35 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -425,12 +425,20 @@ def to_cmd_str(cmd): # -1 means reading until EOF read_size = 128 if exit_code is None else -1 - more_stdout = proc.stdout.read1(read_size) or b'' - stdout += more_stdout + # get output as long as output is available; + # note: can't use proc.stdout.read without read_size argument, + # since that will always wait until EOF + more_stdout = True + while more_stdout: + more_stdout = proc.stdout.read(read_size) or b'' + stdout += more_stdout # note: we assume that there won't be any questions in stderr output if split_stderr: - stderr += proc.stderr.read1(read_size) or b'' + more_stderr = True + while more_stdout: + more_stderr = proc.stderr.read(read_size) or b'' + stderr += more_stderr if qa_patterns: if _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): From c090f49056eccf165e3a493c9ad0c8b19d210817 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Apr 2024 21:25:57 +0200 Subject: [PATCH 194/430] fix while loop to get all available stderr output --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 7fa8806a35..2d7494828e 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -436,7 +436,7 @@ def to_cmd_str(cmd): # note: we assume that there won't be any questions in stderr output if split_stderr: more_stderr = True - while more_stdout: + while more_stderr: more_stderr = proc.stderr.read(read_size) or b'' stderr += more_stderr From a508adbe5edf3f77e378bff4c37e5dba044cc6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Fri, 5 Apr 2024 00:49:31 +0200 Subject: [PATCH 195/430] Make module-extensions true by default --- easybuild/tools/config.py | 2 +- easybuild/tools/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 9791eb3dcc..b7d31f2883 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -292,7 +292,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'install_latest_eb_release', 'logtostdout', 'minimal_toolchains', - 'module_extensions', 'module_only', 'package', 'parallel_extensions_install', @@ -328,6 +327,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'lib64_fallback_sanity_check', 'lib64_lib_symlink', 'map_toolchains', + 'module_extensions', 'modules_tool_version_check', 'mpi_tests', 'pre_create_installdir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 54e9b81982..a7c30a4d30 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -581,7 +581,7 @@ def config_options(self): "(implies recursive unloading of modules).", None, 'store_true', False), 'module-extensions': ("Include 'extensions' statement in generated module file (Lua syntax only)", - None, 'store_true', False), + None, 'store_true', True), 'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS), 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, sorted(avail_module_generators().keys())), From 3170d9fe5e70715a58e425b5d16cacec4bd2600e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sat, 6 Apr 2024 04:06:39 +0200 Subject: [PATCH 196/430] Simplify and fix module description generators Old method needlessly involved template resolutions, duplicating the template resolving code, but only partially and would break if anything but the 4 most common templates were used. --- easybuild/tools/module_generator.py | 53 ++++++++--------------------- 1 file changed, 15 insertions(+), 38 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index c537003de5..f4565b0950 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -816,20 +816,19 @@ def get_description(self, conflict=True): """ Generate a description. """ - txt = '\n'.join([ + lines = [ "proc ModulesHelp { } {", " puts stderr {%s" % re.sub(r'([{}\[\]])', r'\\\1', self._generate_help_text()), " }", '}', '', - ]) - - lines = [ - '%(whatis_lines)s', - '', - "set root %(installdir)s", ] + lines.extend([ + "module-whatis {%s}" % re.sub(r'([{}\[\]])', r'\\\1', line) + for line in self._generate_whatis_lines() + ]) + if self.app.cfg['moduleloadnoconflict']: cond_unload = self.conditional_statement(self.is_loaded('%(name)s'), "module unload %(name)s") lines.extend([ @@ -845,18 +844,9 @@ def get_description(self, conflict=True): # - 'conflict Compiler/GCC/4.8.2/OpenMPI' for 'Compiler/GCC/4.8.2/OpenMPI/1.6.4' lines.extend(['', "conflict %s" % os.path.dirname(self.app.short_mod_name)]) - whatis_lines = [ - "module-whatis {%s}" % re.sub(r'([{}\[\]])', r'\\\1', line) - for line in self._generate_whatis_lines() - ] - txt += '\n'.join([''] + lines + ['']) % { - 'name': self.app.name, - 'version': self.app.version, - 'whatis_lines': '\n'.join(whatis_lines), - 'installdir': self.app.installdir, - } + lines.extend(['', "set root %(installdir)s"]) - return txt + return '\n'.join([''] + lines + ['']) def getenv_cmd(self, envvar, default=None): """ @@ -1261,18 +1251,17 @@ def get_description(self, conflict=True): """ Generate a description. """ - txt = '\n'.join([ + lines = [ 'help(%s%s' % (self.START_STR, self.check_str(self._generate_help_text())), '%s)' % self.END_STR, '', - ]) - - lines = [ - "%(whatis_lines)s", - '', - 'local root = "%(installdir)s"', ] + for line in self._generate_whatis_lines(): + lines.append("whatis(%s%s%s)" % (self.START_STR, self.check_str(line), self.END_STR)) + + lines.extend(['', 'local root = "%(installdir)s"']) + if self.app.cfg['moduleloadnoconflict']: self.log.info("Nothing to do to ensure no conflicts can occur on load when using Lua modules files/Lmod") @@ -1280,10 +1269,6 @@ def get_description(self, conflict=True): # conflict on 'name' part of module name (excluding version part at the end) lines.extend(['', 'conflict("%s")' % os.path.dirname(self.app.short_mod_name)]) - whatis_lines = [] - for line in self._generate_whatis_lines(): - whatis_lines.append("whatis(%s%s%s)" % (self.START_STR, self.check_str(line), self.END_STR)) - if build_option('module_extensions'): extensions_list = self._generate_extensions_list() @@ -1294,15 +1279,7 @@ def get_description(self, conflict=True): # https://github.com/TACC/Lmod/issues/428 lines.extend(['', self.conditional_statement(self.check_version("8", "2", "8"), extensions_stmt)]) - txt += '\n'.join([''] + lines + ['']) % { - 'name': self.app.name, - 'version': self.app.version, - 'whatis_lines': '\n'.join(whatis_lines), - 'installdir': self.app.installdir, - 'homepage': self.app.cfg['homepage'], - } - - return txt + return '\n'.join([''] + lines + ['']) def getenv_cmd(self, envvar, default=None): """ From 22862e1cd66d29a129ae3b2c61f72f751db6128e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Mon, 8 Apr 2024 10:54:50 +0000 Subject: [PATCH 197/430] Use installdir directly instead of introducing a template variable --- easybuild/tools/module_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index f4565b0950..8b829c75b7 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -844,7 +844,7 @@ def get_description(self, conflict=True): # - 'conflict Compiler/GCC/4.8.2/OpenMPI' for 'Compiler/GCC/4.8.2/OpenMPI/1.6.4' lines.extend(['', "conflict %s" % os.path.dirname(self.app.short_mod_name)]) - lines.extend(['', "set root %(installdir)s"]) + lines.extend(['', "set root " + self.app.installdir]) return '\n'.join([''] + lines + ['']) @@ -1260,7 +1260,7 @@ def get_description(self, conflict=True): for line in self._generate_whatis_lines(): lines.append("whatis(%s%s%s)" % (self.START_STR, self.check_str(line), self.END_STR)) - lines.extend(['', 'local root = "%(installdir)s"']) + lines.extend(['', 'local root = "%s"' % self.app.installdir]) if self.app.cfg['moduleloadnoconflict']: self.log.info("Nothing to do to ensure no conflicts can occur on load when using Lua modules files/Lmod") From 0e156571d78d19eea7e88f98a429a981910bb8f1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Apr 2024 21:27:19 +0200 Subject: [PATCH 198/430] fix showing end of stdout output when no question/wait patterns match was found --- easybuild/tools/run.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 2d7494828e..0b8441b6fa 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -209,13 +209,14 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): """ match_found = False + stdout_end = stdout.decode(errors='ignore')[-1000:] for question, answers in qa_patterns: # allow extra whitespace at the end question += r'[\s\n]*$' regex = re.compile(question.encode()) res = regex.search(stdout) if res: - _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout[:1000]}") + _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout_end}") # if answer is specified as a list, we take the first item as current answer, # and add it to the back of the list (so we cycle through answers) if isinstance(answers, list): @@ -249,11 +250,12 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): regex = re.compile(pattern.encode()) if regex.search(stdout): _log.info(f"Found match for wait pattern '{pattern}'") - _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout[:1000]}") + _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout_end}") match_found = True break else: _log.info("No match found for question wait patterns") + _log.debug(f"No match found in question/wait patterns at end of stdout: {stdout_end}") return match_found @@ -444,7 +446,6 @@ def to_cmd_str(cmd): if _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): time_no_match = 0 else: - _log.debug(f"No match found in question/wait patterns at end of stdout: {stdout[:1000]}") # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs if time_no_match > qa_timeout: From ec288bc1edb5b94680fb3dcbae1774a01f43b745 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 9 Apr 2024 11:30:21 +0200 Subject: [PATCH 199/430] add debug log statement when more stdout output is obtained in run_shell_cmd --- easybuild/tools/run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 0b8441b6fa..21eac92a70 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -433,6 +433,7 @@ def to_cmd_str(cmd): more_stdout = True while more_stdout: more_stdout = proc.stdout.read(read_size) or b'' + _log.debug(f"Obtained more stdout: {more_stdout}") stdout += more_stdout # note: we assume that there won't be any questions in stderr output From 0863b12e6a2b133525074c01f550144634292a20 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 9 Apr 2024 11:45:23 +0200 Subject: [PATCH 200/430] stop checking run.errors_found_in_log, which is only updated by now deprecated parse_log_for_error (used by run_cmd + run_cmd_qa) --- easybuild/framework/easyblock.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 72fa56cab8..55a924dd4d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4246,7 +4246,6 @@ def build_and_install_one(ecdict, init_env): # restore original environment, and then sanitize it _log.info("Resetting environment") - run.errors_found_in_log = 0 restore_env(init_env) sanitize_env() @@ -4449,11 +4448,6 @@ def ensure_writable_log_dir(log_dir): req_time = time2str(end_timestamp - start_timestamp) print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent) - # check for errors - if run.errors_found_in_log > 0: - _log.warning("%d possible error(s) were detected in the " - "build logs, please verify the build.", run.errors_found_in_log) - if app.postmsg: print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent) From 423213676391126db05cbcf242210dce9823ba1d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Apr 2024 17:35:59 +0200 Subject: [PATCH 201/430] Fix typo in patch_step logging --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d2727d61ee..b17806b7c2 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2607,7 +2607,7 @@ def patch_step(self, beginpath=None, patches=None): copy_patch = 'copy' in patch and 'sourcepath' not in patch self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s", - srcind, level, srcpathsuffix, copy) + srcind, level, srcpathsuffix, copy_patch) if beginpath is None: try: From bdba1c2f85bccc9888a2f0fcb9a3546e1f0cfcb0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Apr 2024 17:12:33 +0200 Subject: [PATCH 202/430] Improve behavior when using extension with 'nosource:True' --- easybuild/framework/extensioneasyblock.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 69c824fe7e..86ea151cbb 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -123,7 +123,9 @@ def _set_start_dir(self): self.log.debug("Using extension start dir: %s", ext_start_dir) self.cfg['start_dir'] = ext_start_dir self.cfg.template_values['start_dir'] = ext_start_dir - elif ext_start_dir is None: + return ext_start_dir + + if ext_start_dir is None: # This may be on purpose, e.g. for Python WHL files which do not get extracted self.log.debug("Start dir is not set.") else: @@ -137,7 +139,10 @@ def run(self, unpack_src=False): """Common operations for extensions: unpacking sources, patching, ...""" # unpack file if desired - if unpack_src: + if self.options.get('nosource', False): + # If no source wanted use the start_dir from the main EC + self.ext_dir = self.master.start_dir + elif unpack_src: targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name)) self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options, change_into_dir=False, cmd=self.src_extract_cmd) @@ -146,10 +151,9 @@ def run(self, unpack_src=False): # because start_dir value is usually a relative path (if it is set) change_dir(self.ext_dir) - self._set_start_dir() + start_dir = self._set_start_dir() + if start_dir: change_dir(self.start_dir) - else: - self._set_start_dir() # patch if needed EasyBlock.patch_step(self, beginpath=self.ext_dir) From 138d5bea9a4df2143fa1110f16d63a2619543708 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Apr 2024 18:17:48 +0200 Subject: [PATCH 203/430] Throw error if non-existant startdir was provided Improve the current error that we couldn't change into that directory. --- easybuild/framework/extensioneasyblock.py | 16 ++++++++-------- test/framework/easyblock.py | 5 +---- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 86ea151cbb..efd7c349f7 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -123,15 +123,15 @@ def _set_start_dir(self): self.log.debug("Using extension start dir: %s", ext_start_dir) self.cfg['start_dir'] = ext_start_dir self.cfg.template_values['start_dir'] = ext_start_dir - return ext_start_dir - - if ext_start_dir is None: + elif ext_start_dir is None: # This may be on purpose, e.g. for Python WHL files which do not get extracted self.log.debug("Start dir is not set.") - else: + elif self.start_dir: # non-existing start dir means wrong input from user - warn_msg = "Provided start dir (%s) for extension %s does not exist: %s" % (self.start_dir, self.name, - ext_start_dir) + raise EasyBuildError("Provided start dir (%s) for extension %s does not exist: %s", + self.start_dir, self.name, ext_start_dir) + else: + warn_msg = 'Failed to determine start dir for extension %s: %s' % (self.name, ext_start_dir) self.log.warning(warn_msg) print_warning(warn_msg, silent=build_option('silent')) @@ -151,8 +151,8 @@ def run(self, unpack_src=False): # because start_dir value is usually a relative path (if it is set) change_dir(self.ext_dir) - start_dir = self._set_start_dir() - if start_dir: + self._set_start_dir() + if self.start_dir: change_dir(self.start_dir) # patch if needed diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e6b54e0bc8..6ccd971141 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2257,11 +2257,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): 'start_dir': 'nonexistingdir'}), ] with self.mocked_stdout_stderr(): - err_pattern = "Failed to change from .*barbar/barbar-0.0 to nonexistingdir.*" + err_pattern = r"Provided start dir \(nonexistingdir\) for extension barbar does not exist:.*" self.assertErrorRegex(EasyBuildError, err_pattern, check_ext_start_dir, 'whatever') - stderr = self.get_stderr() - warning_pattern = "WARNING: Provided start dir (nonexistingdir) for extension barbar does not exist" - self.assertIn(warning_pattern, stderr) # No error when using relative path in non-extracted source for some reason ec['ec']['exts_list'] = [ From 948ee8d661bff7d2f28ba111e5706a08fa845a13 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Apr 2024 18:19:35 +0200 Subject: [PATCH 204/430] Fix tests for changed behavior and add test for nosource --- test/framework/easyblock.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6ccd971141..7ab5674b97 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2200,11 +2200,16 @@ def test_extension_set_start_dir(self): cwd = os.getcwd() self.assertExists(cwd) - def check_ext_start_dir(expected_start_dir, unpack_src=True): + def check_ext_start_dir(expected_start_dir, unpack_src=True, parent_startdir=None): """Check start dir.""" # make sure we're in an existing directory at the start change_dir(cwd) + eb = EasyBlock(ec['ec']) + if not os.path.exists(eb.builddir): + eb.make_builddir() # Required to exist for samefile + eb.cfg['start_dir'] = parent_startdir + eb.extensions_step(fetch=True, install=False) # extract sources of the extension ext = eb.ext_instances[-1] @@ -2212,6 +2217,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): if expected_start_dir is None: self.assertIsNone(ext.start_dir) + # Without a start dir we don't change the CWD + self.assertEqual(os.getcwd(), cwd) else: self.assertTrue(os.path.isabs(ext.start_dir)) if ext.start_dir != os.sep: @@ -2221,14 +2228,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): else: abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir) self.assertEqual(ext.start_dir, abs_expected_start_dir) - if not os.path.exists(eb.builddir): - eb.make_builddir() # Required to exist for samefile self.assertTrue(os.path.samefile(ext.start_dir, abs_expected_start_dir)) - if unpack_src: self.assertTrue(os.path.samefile(os.getcwd(), abs_expected_start_dir)) - else: - # When not unpacking we don't change the CWD - self.assertEqual(os.getcwd(), cwd) remove_dir(eb.builddir) ec['ec']['exts_defaultclass'] = 'DummyExtension' @@ -2288,6 +2289,15 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): check_ext_start_dir(os.sep, unpack_src=False) self.assertFalse(self.get_stderr()) + # Go to ECs start dir if nosource is used + ec['ec']['exts_list'] = [ + ('barbar', '0.0', { + 'nosource': True}), + ] + with self.mocked_stdout_stderr(): + check_ext_start_dir(self.test_prefix, parent_startdir=self.test_prefix) + self.assertFalse(self.get_stderr()) + def test_prepare_step(self): """Test prepare step (setting up build environment).""" test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') From 9c3d6ec1086ac42cb67e027f8eda1de27e82ad1b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 14 Apr 2024 15:09:18 +0200 Subject: [PATCH 205/430] fix imports --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/run.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 55a924dd4d..bf9c939231 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -68,7 +68,7 @@ from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.framework.extension import Extension, resolve_exts_filter_template -from easybuild.tools import LooseVersion, config, run +from easybuild.tools import LooseVersion, config from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index cfc9d6b408..7866357526 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -46,13 +46,14 @@ import subprocess import sys import tempfile +import time from collections import namedtuple from datetime import datetime # import deprecated functions so they can still be imported from easybuild.tools.run, for now from easybuild._deprecated import check_async_cmd, check_log_for_errors, complete_cmd, extract_errors_from_log # noqa -from easybuild._deprecated import get_output_from_process, parse_cmd_output, parse_log_for_error, run_cmd # noqa -from easybuild._deprecated import run_cmd_cache, run_cmd_qa # noqa +from easybuild._deprecated import get_output_from_process, parse_cmd_output, parse_log_for_error # noqa +from easybuild._deprecated import run_cmd, run_cmd_qa # noqa try: # get_native_id is only available in Python >= 3.8 @@ -182,9 +183,6 @@ def cache_aware_func(cmd, *args, **kwargs): return cache_aware_func -run_shell_cmd_cache = run_cmd_cache - - def fileprefix_from_cmd(cmd, allowed_chars=False): """ Simplify the cmd to only the allowed_chars we want in a filename From 04413a550b406f99fc81af47aeae6a629ec6c18b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 14 Apr 2024 15:27:59 +0200 Subject: [PATCH 206/430] fix use of IGNORE/ERROR/WARN in filetools tests --- test/framework/filetools.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 16923c55d2..064d8c09df 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -47,10 +47,9 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from urllib import request -from easybuild.tools import run import easybuild.tools.filetools as ft from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option +from easybuild.tools.config import IGNORE, ERROR, WARN, build_option, update_build_option from easybuild.tools.multidiff import multidiff @@ -1442,7 +1441,7 @@ def test_apply_regex_substitutions(self): # passing empty list of substitions is a no-op ft.write_file(testfile, testtxt) - ft.apply_regex_substitutions(testfile, [], on_missing_match=run.IGNORE) + ft.apply_regex_substitutions(testfile, [], on_missing_match=IGNORE) new_testtxt = ft.read_file(testfile) self.assertEqual(new_testtxt, testtxt) @@ -1452,17 +1451,17 @@ def test_apply_regex_substitutions(self): error_pat = 'Nothing found to replace in %s' % testfile # Error self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, testfile, regex_subs_no_match, - on_missing_match=run.ERROR) + on_missing_match=ERROR) # Warn with self.log_to_testlogfile(): - ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.WARN) + ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=WARN) logtxt = ft.read_file(self.logfile) self.assertIn('WARNING ' + error_pat, logtxt) # Ignore with self.log_to_testlogfile(): - ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.IGNORE) + ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=IGNORE) logtxt = ft.read_file(self.logfile) self.assertIn('INFO ' + error_pat, logtxt) From 72704a78915de8dc4ad87160ed99594666ac8110 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 14 Apr 2024 15:33:24 +0200 Subject: [PATCH 207/430] allow triggering deprecated behaviour in tests for run_cmd and run_cmd_qa --- test/framework/lib.py | 8 ++-- test/framework/run.py | 108 ++++++++++++++++++++++++++++++++---------- 2 files changed, 86 insertions(+), 30 deletions(-) diff --git a/test/framework/lib.py b/test/framework/lib.py index b3deff6070..bcc4c8e7b4 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -74,14 +74,14 @@ def test_run_cmd(self): error_pattern = r"Undefined build option: .*" error_pattern += r" Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)" - self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd, "echo hello") + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd, "echo hello") self.configure() # run_cmd works fine if set_up_configuration was called first - self.mock_stdout(True) - (out, ec) = run_cmd("echo hello") - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + (out, ec) = run_cmd("echo hello") self.assertEqual(ec, 0) self.assertEqual(out, 'hello\n') diff --git a/test/framework/run.py b/test/framework/run.py index f1ee6955ec..2e0837e225 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -142,6 +142,10 @@ def get_proc(cmd, asynchronous=False): def test_run_cmd(self): """Basic test for run_cmd function.""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + with self.mocked_stdout_stderr(): (out, ec) = run_cmd("echo hello") self.assertEqual(out, "hello\n") @@ -217,6 +221,10 @@ def test_fileprefix_from_cmd(self): def test_run_cmd_log(self): """Test logging of executed commands.""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) @@ -300,6 +308,10 @@ def test_run_shell_cmd_log(self): def test_run_cmd_negative_exit_code(self): """Test run_cmd function with command that has negative exit code.""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + # define signal handler to call in case run_cmd takes too long def handler(signum, _): raise RuntimeError("Signal handler called with signal %s" % signum) @@ -433,6 +445,10 @@ def handler(signum, _): def test_run_cmd_bis(self): """More 'complex' test for run_cmd function.""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + # a more 'complex' command to run, make sure all required output is there with self.mocked_stdout_stderr(): (out, ec) = run_cmd("for j in `seq 1 3`; do for i in `seq 1 100`; do echo hello; done; sleep 1.4; done") @@ -453,6 +469,10 @@ def test_run_cmd_work_dir(self): """ Test running command in specific directory with run_cmd function. """ + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + orig_wd = os.getcwd() self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) @@ -493,6 +513,10 @@ def test_run_shell_cmd_work_dir(self): def test_run_cmd_log_output(self): """Test run_cmd with log_output enabled""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + with self.mocked_stdout_stderr(): (out, ec) = run_cmd("seq 1 100", log_output=True) self.assertEqual(ec, 0) @@ -548,6 +572,9 @@ def test_run_shell_cmd_split_stderr(self): def test_run_cmd_trace(self): """Test run_cmd in trace mode, and with tracing disabled.""" + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + pattern = [ r"^ >> running command:", r"\t\[started at: .*\]", @@ -567,7 +594,7 @@ def test_run_cmd_trace(self): self.mock_stderr(False) self.assertEqual(out, 'hello\n') self.assertEqual(ec, 0) - self.assertEqual(stderr, '') + self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality")) regex = re.compile('\n'.join(pattern)) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) @@ -582,7 +609,7 @@ def test_run_cmd_trace(self): self.mock_stderr(False) self.assertEqual(out, 'hello\n') self.assertEqual(ec, 0) - self.assertEqual(stderr, '') + self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality")) self.assertEqual(stdout, '') init_config(build_options={'trace': True}) @@ -597,7 +624,7 @@ def test_run_cmd_trace(self): self.mock_stderr(False) self.assertEqual(out, 'hello') self.assertEqual(ec, 0) - self.assertEqual(stderr, '') + self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality")) pattern.insert(3, r"\t\[input: hello\]") pattern[-2] = "\tcat" regex = re.compile('\n'.join(pattern)) @@ -614,7 +641,7 @@ def test_run_cmd_trace(self): self.mock_stderr(False) self.assertEqual(out, 'hello') self.assertEqual(ec, 0) - self.assertEqual(stderr, '') + self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality")) self.assertEqual(stdout, '') # trace output can be disabled on a per-command basis @@ -631,7 +658,7 @@ def test_run_cmd_trace(self): self.assertEqual(out, 'hello\n') self.assertEqual(ec, 0) self.assertEqual(stdout, '') - self.assertEqual(stderr, '') + self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality")) def test_run_shell_cmd_trace(self): """Test run_shell_cmd function in trace mode, and with tracing disabled.""" @@ -963,6 +990,9 @@ def test_run_shell_cmd_qa_log(self): def test_run_cmd_qa_trace(self): """Test run_cmd under --trace""" + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + # --trace is enabled by default self.mock_stdout(True) self.mock_stderr(True) @@ -989,7 +1019,7 @@ def test_run_cmd_qa_trace(self): self.mock_stdout(False) self.mock_stderr(False) self.assertEqual(stdout, '') - self.assertEqual(stderr, '') + self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality")) def test_run_shell_cmd_qa_trace(self): """Test run_shell_cmd with qa_patterns under --trace""" @@ -1071,12 +1101,20 @@ def test_run_shell_cmd_qa_answers(self): def test_run_cmd_simple(self): """Test return value for run_cmd in 'simple' mode.""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + with self.mocked_stdout_stderr(): self.assertEqual(True, run_cmd("echo hello", simple=True)) self.assertEqual(False, run_cmd("exit 1", simple=True, log_all=False, log_ok=False)) def test_run_cmd_cache(self): """Test caching for run_cmd""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + with self.mocked_stdout_stderr(): (first_out, ec) = run_cmd("ulimit -u") self.assertEqual(ec, 0) @@ -1164,6 +1202,10 @@ def test_parse_log_error(self): def test_run_cmd_dry_run(self): """Test use of run_cmd function under (extended) dry run.""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + build_options = { 'extended_dry_run': True, 'silent': False, @@ -1172,43 +1214,38 @@ def test_run_cmd_dry_run(self): cmd = "somecommand foo 123 bar" - self.mock_stdout(True) - run_cmd(cmd) - stdout = self.get_stdout() - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + run_cmd(cmd) + stdout = self.get_stdout() expected = """ running command "somecommand foo 123 bar"\n""" self.assertIn(expected, stdout) # check disabling 'verbose' - self.mock_stdout(True) - run_cmd("somecommand foo 123 bar", verbose=False) - stdout = self.get_stdout() - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + run_cmd("somecommand foo 123 bar", verbose=False) + stdout = self.get_stdout() self.assertNotIn(expected, stdout) # check forced run_cmd outfile = os.path.join(self.test_prefix, 'cmd.out') self.assertNotExists(outfile) - self.mock_stdout(True) - run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True) - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True) self.assertExists(outfile) self.assertEqual(read_file(outfile), "This is always echoed\n") # Q&A commands - self.mock_stdout(True) - run_shell_cmd("some_qa_cmd", qa_patterns=[('question1', 'answer1')]) - stdout = self.get_stdout() - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + run_shell_cmd("some_qa_cmd", qa_patterns=[('question1', 'answer1')]) + stdout = self.get_stdout() expected = """ running interactive shell command "some_qa_cmd"\n""" self.assertIn(expected, stdout) - self.mock_stdout(True) - run_cmd_qa("some_qa_cmd", {'question1': 'answer1'}) - stdout = self.get_stdout() - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + run_cmd_qa("some_qa_cmd", {'question1': 'answer1'}) + stdout = self.get_stdout() expected = """ running interactive command "some_qa_cmd"\n""" self.assertIn(expected, stdout) @@ -1264,6 +1301,10 @@ def test_run_shell_cmd_dry_run(self): def test_run_cmd_list(self): """Test run_cmd with command specified as a list rather than a string""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + cmd = ['/bin/sh', '-c', "echo hello"] with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, "When passing cmd as a list then `shell` must be set explictely!", @@ -1275,6 +1316,10 @@ def test_run_cmd_list(self): def test_run_cmd_script(self): """Testing use of run_cmd with shell=False to call external scripts""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + py_test_script = os.path.join(self.test_prefix, 'test.py') write_file(py_test_script, '\n'.join([ '#!%s' % sys.executable, @@ -1313,6 +1358,10 @@ def test_run_shell_cmd_no_bash(self): def test_run_cmd_stream(self): """Test use of run_cmd with streaming output.""" + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + self.mock_stdout(True) self.mock_stderr(True) (out, ec) = run_cmd("echo hello", stream_output=True) @@ -1324,7 +1373,7 @@ def test_run_cmd_stream(self): self.assertEqual(ec, 0) self.assertEqual(out, "hello\n") - self.assertEqual(stderr, '') + self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality")) expected = [ "== (streaming) output for command 'echo hello':", "hello", @@ -1370,6 +1419,9 @@ def test_run_shell_cmd_stream(self): def test_run_cmd_async(self): """Test asynchronously running of a shell command via run_cmd + complete_cmd.""" + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + os.environ['TEST'] = 'test123' test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" @@ -1604,6 +1656,10 @@ def test_run_cmd_with_hooks(self): """ Test running command with run_cmd with pre/post run_shell_cmd hooks in place. """ + + # use of run_cmd is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + cwd = os.getcwd() hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') From a64e574407228aa928d1ccb8f6b3ef9a93c73fb7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 14 Apr 2024 16:04:33 +0200 Subject: [PATCH 208/430] fix broken tests --- test/framework/build_log.py | 8 ++++---- test/framework/config.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 2bb705b333..5cba105fec 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -118,9 +118,9 @@ def test_easybuildlog(self): more_info = "see https://docs.easybuild.io/deprecated-functionality/ for more information" expected_stderr = '\n\n'.join([ - "\nWARNING: Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info, - "\nWARNING: Deprecated functionality, will no longer work in v2.0: onemorewarning", - "\nWARNING: Deprecated functionality, will no longer work in v2.0: lastwarning", + "\nWARNING: Deprecated functionality, will no longer work in EasyBuild v10000001: anotherwarning; " + more_info, + "\nWARNING: Deprecated functionality, will no longer work in EasyBuild v2.0: onemorewarning", + "\nWARNING: Deprecated functionality, will no longer work in EasyBuild v2.0: lastwarning", ]) + '\n\n' self.assertEqual(stderr, expected_stderr) @@ -183,7 +183,7 @@ def test_easybuildlog(self): self.mock_stderr(False) logtxt = read_file(tmplog) expected_logtxt = '\n'.join([ - "[WARNING] :: Deprecated functionality, will no longer work in v10000001: ", + "[WARNING] :: Deprecated functionality, will no longer work in EasyBuild v10000001: ", "this is just a test", "(see URLGOESHERE for more information)", ]) diff --git a/test/framework/config.py b/test/framework/config.py index 127fe1f538..a6bd5a1233 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -40,10 +40,10 @@ import easybuild.tools.options as eboptions from easybuild.tools import run from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import ERROR, IGNORE, WARN, BuildOptions, ConfigurationVariables from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath from easybuild.tools.config import install_path, log_file_format, log_path, source_paths from easybuild.tools.config import update_build_option, update_build_options -from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options from easybuild.tools.filetools import copy_dir, mkdir, write_file from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX @@ -580,9 +580,9 @@ def test_flex_robot_paths(self): def test_strict(self): """Test use of --strict.""" # check default - self.assertEqual(build_option('strict'), run.WARN) + self.assertEqual(build_option('strict'), WARN) - for strict_str, strict_val in [('error', run.ERROR), ('ignore', run.IGNORE), ('warn', run.WARN)]: + for strict_str, strict_val in [('error', ERROR), ('ignore', IGNORE), ('warn', WARN)]: options = init_config(args=['--strict=%s' % strict_str]) init_config(build_options={'strict': options.strict}) self.assertEqual(build_option('strict'), strict_val) From 6831843a46aab2780fc9c9fe0eb5fbe67054eab4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 14 Apr 2024 18:15:51 +0200 Subject: [PATCH 209/430] fix long line in test_easybuildlog --- test/framework/build_log.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 5cba105fec..838d782a9e 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -117,10 +117,11 @@ def test_easybuildlog(self): self.mock_stderr(False) more_info = "see https://docs.easybuild.io/deprecated-functionality/ for more information" + common_warning = "\nWARNING: Deprecated functionality, will no longer work in" expected_stderr = '\n\n'.join([ - "\nWARNING: Deprecated functionality, will no longer work in EasyBuild v10000001: anotherwarning; " + more_info, - "\nWARNING: Deprecated functionality, will no longer work in EasyBuild v2.0: onemorewarning", - "\nWARNING: Deprecated functionality, will no longer work in EasyBuild v2.0: lastwarning", + common_warning + " EasyBuild v10000001: anotherwarning; " + more_info, + common_warning + " EasyBuild v2.0: onemorewarning", + common_warning + " EasyBuild v2.0: lastwarning", ]) + '\n\n' self.assertEqual(stderr, expected_stderr) From cd7750a946b8b5cbe34262345346ac4d1e98f6dc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 14 Apr 2024 18:32:13 +0200 Subject: [PATCH 210/430] consider both easybuild-framework*.tar.gz and easybuild_framework*.tar.gz in CI workflows --- .github/workflows/container_tests.yml | 2 +- .github/workflows/container_tests_apptainer.yml | 2 +- .github/workflows/eb_command.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index ef310a3816..540fafcea1 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -73,7 +73,7 @@ jobs: python setup.py sdist ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA - pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz - name: run test diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 77d2a4a395..f7040000f9 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -73,7 +73,7 @@ jobs: python setup.py sdist ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA - pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz - name: run test diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index ff9a5eaa4f..ec4907d28c 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -68,7 +68,7 @@ jobs: python setup.py sdist ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA - pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz - name: run tests for 'eb' command env: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index c79b515e42..07ac74fe1a 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -140,7 +140,7 @@ jobs: python setup.py sdist ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA - pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz - name: run test suite env: From dc9c218b07616ba6995bd6ac9dfdce32792b1cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 20 Mar 2024 00:36:49 +0100 Subject: [PATCH 211/430] Attempt command logging --- easybuild/tools/run.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 21eac92a70..3b9436d50c 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -43,6 +43,7 @@ import os import re import signal +import shlex import shutil import string import subprocess @@ -203,6 +204,22 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): return ''.join([c for c in cmd if c in allowed_chars]) +def save_cmd(cmd, work_dir, env): + cmd_name = fileprefix_from_cmd(os.path.basename(to_cmd_str(cmd).split(' ')[0])) + full_env = os.environ.copy() + full_env.update(env) + + with tempfile.NamedTemporaryFile(prefix=f"{cmd_name}-", suffix=".sh", delete=False) as fid: + fid.write(f'cd "{work_dir}"\n') + fid.write(f'history -s "{shlex.quote(cmd)}"\n') + for key, value in full_env.items(): + fid.write(f'{key}={shlex.quote(value)}\n') + fid.write(f'export PS1="eb-shell> $PS1"\n') + fid.write(f'echo Shell for the command: "{shlex.quote(cmd)}"\n') + fid.write(f'echo Use command history, exit to stop\n') + fid.write(f'bash\n') + + def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): """ Private helper function to try and answer questions raised in interactive shell commands. @@ -393,6 +410,7 @@ def to_cmd_str(cmd): log_msg += f" (via thread with ID {thread_id})" _log.info(log_msg) + save_cmd(cmd, work_dir, env) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, stdin=subprocess.PIPE, cwd=work_dir, env=env, shell=shell, executable=executable) From 75a6070094e7da67d5dfbea4a77ca09bdad85b5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 20 Mar 2024 00:47:39 +0100 Subject: [PATCH 212/430] Use only cmd_str for logging --- easybuild/tools/run.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 3b9436d50c..245163dd22 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -204,18 +204,18 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): return ''.join([c for c in cmd if c in allowed_chars]) -def save_cmd(cmd, work_dir, env): - cmd_name = fileprefix_from_cmd(os.path.basename(to_cmd_str(cmd).split(' ')[0])) +def save_cmd(cmd_str, work_dir, env): + cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) full_env = os.environ.copy() full_env.update(env) with tempfile.NamedTemporaryFile(prefix=f"{cmd_name}-", suffix=".sh", delete=False) as fid: fid.write(f'cd "{work_dir}"\n') - fid.write(f'history -s "{shlex.quote(cmd)}"\n') + fid.write(f'history -s "{shlex.quote(cmd_str)}"\n') for key, value in full_env.items(): fid.write(f'{key}={shlex.quote(value)}\n') fid.write(f'export PS1="eb-shell> $PS1"\n') - fid.write(f'echo Shell for the command: "{shlex.quote(cmd)}"\n') + fid.write(f'echo Shell for the command: "{shlex.quote(cmd_str)}"\n') fid.write(f'echo Use command history, exit to stop\n') fid.write(f'bash\n') @@ -410,7 +410,7 @@ def to_cmd_str(cmd): log_msg += f" (via thread with ID {thread_id})" _log.info(log_msg) - save_cmd(cmd, work_dir, env) + save_cmd(cmd_str, work_dir, env) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, stdin=subprocess.PIPE, cwd=work_dir, env=env, shell=shell, executable=executable) From 84ccc735744c03ac64f864a91dbbb389933de539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 20 Mar 2024 00:51:25 +0100 Subject: [PATCH 213/430] Check for env == None --- easybuild/tools/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 245163dd22..11412d0dcf 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -207,7 +207,8 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): def save_cmd(cmd_str, work_dir, env): cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) full_env = os.environ.copy() - full_env.update(env) + if env is not None: + full_env.update(env) with tempfile.NamedTemporaryFile(prefix=f"{cmd_name}-", suffix=".sh", delete=False) as fid: fid.write(f'cd "{work_dir}"\n') From 9cae748b563ebefafe074bba052b2eedb28998a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 20 Mar 2024 13:59:38 +0000 Subject: [PATCH 214/430] Fix utf-8 encoded output for cmd log --- easybuild/tools/run.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 11412d0dcf..a407756ccd 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -211,14 +211,15 @@ def save_cmd(cmd_str, work_dir, env): full_env.update(env) with tempfile.NamedTemporaryFile(prefix=f"{cmd_name}-", suffix=".sh", delete=False) as fid: - fid.write(f'cd "{work_dir}"\n') - fid.write(f'history -s "{shlex.quote(cmd_str)}"\n') - for key, value in full_env.items(): - fid.write(f'{key}={shlex.quote(value)}\n') - fid.write(f'export PS1="eb-shell> $PS1"\n') - fid.write(f'echo Shell for the command: "{shlex.quote(cmd_str)}"\n') - fid.write(f'echo Use command history, exit to stop\n') - fid.write(f'bash\n') + fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items()).encode('utf-8')) + fid.write('\n'.join([ + f'cd "{work_dir}"', + f'history -s "{shlex.quote(cmd_str)}"', + f'export PS1="eb-shell> $PS1"', + f'echo Shell for the command: "{shlex.quote(cmd_str)}"', + f'echo Use command history, exit to stop', + f'bash', + ]).encode('utf-8')) def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): From aa4b41726b7e1bd7efb7f12d96fffabcbba775b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 20 Mar 2024 14:32:58 +0000 Subject: [PATCH 215/430] Use normal strings instead of f-strings --- easybuild/tools/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index a407756ccd..a5019ac6f0 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -215,10 +215,10 @@ def save_cmd(cmd_str, work_dir, env): fid.write('\n'.join([ f'cd "{work_dir}"', f'history -s "{shlex.quote(cmd_str)}"', - f'export PS1="eb-shell> $PS1"', f'echo Shell for the command: "{shlex.quote(cmd_str)}"', - f'echo Use command history, exit to stop', - f'bash', + 'echo Use command history, exit to stop', + 'export PS1="eb-shell> $PS1"', + 'bash', ]).encode('utf-8')) From e3e582c9a0687db237d615efdbfb31263032e1f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Fri, 29 Mar 2024 17:47:01 +0000 Subject: [PATCH 216/430] Move save_cmd to output directory --- easybuild/tools/run.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index a5019ac6f0..d0c661326f 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -204,14 +204,13 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): return ''.join([c for c in cmd if c in allowed_chars]) -def save_cmd(cmd_str, work_dir, env): - cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) +def save_cmd(cmd_str, work_dir, env, filename): full_env = os.environ.copy() if env is not None: full_env.update(env) - with tempfile.NamedTemporaryFile(prefix=f"{cmd_name}-", suffix=".sh", delete=False) as fid: - fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items()).encode('utf-8')) + with open(filename, 'w') as fid: + fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items())) fid.write('\n'.join([ f'cd "{work_dir}"', f'history -s "{shlex.quote(cmd_str)}"', @@ -219,7 +218,7 @@ def save_cmd(cmd_str, work_dir, env): 'echo Use command history, exit to stop', 'export PS1="eb-shell> $PS1"', 'bash', - ]).encode('utf-8')) + ])) def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): @@ -352,11 +351,14 @@ def to_cmd_str(cmd): os.makedirs(toptmpdir, exist_ok=True) cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') + cmd_log_fp = os.path.join(tmpdir, 'cmd.sh') + _log.info(f'run_shell_cmd: command environment of "{cmd_str}" will be saved to {cmd_log_fp}') + save_cmd(cmd_str, work_dir, env, cmd_log_fp) cmd_out_fp = os.path.join(tmpdir, 'out.txt') - _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') + _log.info(f'run_shell_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') if split_stderr: cmd_err_fp = os.path.join(tmpdir, 'err.txt') - _log.info(f'run_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') + _log.info(f'run_shell_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') else: cmd_err_fp = None else: @@ -412,7 +414,6 @@ def to_cmd_str(cmd): log_msg += f" (via thread with ID {thread_id})" _log.info(log_msg) - save_cmd(cmd_str, work_dir, env) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, stdin=subprocess.PIPE, cwd=work_dir, env=env, shell=shell, executable=executable) From 30d300c075085d720b3178df7ba51e1c590488b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 7 Apr 2024 15:31:07 +0200 Subject: [PATCH 217/430] Remove double qouting in save_cmd, add missing newline --- easybuild/tools/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index d0c661326f..0845b0e9db 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -212,9 +212,9 @@ def save_cmd(cmd_str, work_dir, env, filename): with open(filename, 'w') as fid: fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items())) fid.write('\n'.join([ - f'cd "{work_dir}"', - f'history -s "{shlex.quote(cmd_str)}"', - f'echo Shell for the command: "{shlex.quote(cmd_str)}"', + f'\ncd "{work_dir}"', + f'history -s {shlex.quote(cmd_str)}', + f'echo Shell for the command: {shlex.quote(cmd_str)}', 'echo Use command history, exit to stop', 'export PS1="eb-shell> $PS1"', 'bash', From 4fd99dc9bf6d4b7c1e79d07d158abe19e6451e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 7 Apr 2024 15:45:35 +0200 Subject: [PATCH 218/430] Exclude bash functions, add shebang, add norc --- easybuild/tools/run.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 0845b0e9db..e8695f05e8 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -209,15 +209,18 @@ def save_cmd(cmd_str, work_dir, env, filename): if env is not None: full_env.update(env) + os.chmod(filename, 0o775) with open(filename, 'w') as fid: - fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items())) + fid.write('#!/usr/bin/env bash') + # excludes bash functions (environment variables ending with %) + fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items() if not key.endswith('%')) fid.write('\n'.join([ f'\ncd "{work_dir}"', f'history -s {shlex.quote(cmd_str)}', f'echo Shell for the command: {shlex.quote(cmd_str)}', 'echo Use command history, exit to stop', 'export PS1="eb-shell> $PS1"', - 'bash', + 'bash --norc', ])) From d6eda62174d49698f13a073c028748c6c4947237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 7 Apr 2024 15:49:51 +0200 Subject: [PATCH 219/430] Fix missing parenthesis --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index e8695f05e8..d2ea99ae1d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -213,7 +213,7 @@ def save_cmd(cmd_str, work_dir, env, filename): with open(filename, 'w') as fid: fid.write('#!/usr/bin/env bash') # excludes bash functions (environment variables ending with %) - fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items() if not key.endswith('%')) + fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items() if not key.endswith('%'))) fid.write('\n'.join([ f'\ncd "{work_dir}"', f'history -s {shlex.quote(cmd_str)}', From 66806e1f09bcb96b95b40e9a49b75d65c799ae2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 7 Apr 2024 15:51:01 +0200 Subject: [PATCH 220/430] Move chmod until after file exists --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index d2ea99ae1d..594c15178e 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -209,7 +209,6 @@ def save_cmd(cmd_str, work_dir, env, filename): if env is not None: full_env.update(env) - os.chmod(filename, 0o775) with open(filename, 'w') as fid: fid.write('#!/usr/bin/env bash') # excludes bash functions (environment variables ending with %) @@ -222,6 +221,7 @@ def save_cmd(cmd_str, work_dir, env, filename): 'export PS1="eb-shell> $PS1"', 'bash --norc', ])) + os.chmod(filename, 0o775) def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): From ccd95e03b41af8d31268dd8dc06885eb728a341d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 7 Apr 2024 15:53:12 +0200 Subject: [PATCH 221/430] Add missing linebreak after shebang --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 594c15178e..10a2995f7d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -210,7 +210,7 @@ def save_cmd(cmd_str, work_dir, env, filename): full_env.update(env) with open(filename, 'w') as fid: - fid.write('#!/usr/bin/env bash') + fid.write('#!/usr/bin/env bash\n') # excludes bash functions (environment variables ending with %) fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items() if not key.endswith('%'))) fid.write('\n'.join([ From 5420fa246e4d008a5b4115cbc56ebea8b848dc6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 7 Apr 2024 21:45:34 +0200 Subject: [PATCH 222/430] Split cmd.sh into env.sh and rework bash shell to pick up environment and history --- easybuild/tools/run.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 10a2995f7d..055bddfe5a 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -204,24 +204,31 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): return ''.join([c for c in cmd if c in allowed_chars]) -def save_cmd(cmd_str, work_dir, env, filename): +def save_cmd(cmd_str, work_dir, env, tmpdir): + # Save environment variables in it's own environment file full_env = os.environ.copy() if env is not None: full_env.update(env) + env_fp = os.path.join(tmpdir, 'env.sh') + with open(env_fp, 'w') as fid: + # excludes bash functions (environment variables ending with %) + fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in full_env.items() + if not key.endswith('%'))) - with open(filename, 'w') as fid: + # Make script that sets up bash shell with given environments set. + cmd_fp = os.path.join(tmpdir, 'cmd.sh') + with open(cmd_fp, 'w') as fid: fid.write('#!/usr/bin/env bash\n') - # excludes bash functions (environment variables ending with %) - fid.write('\n'.join(f'{key}={shlex.quote(value)}' for key, value in full_env.items() if not key.endswith('%'))) fid.write('\n'.join([ f'\ncd "{work_dir}"', - f'history -s {shlex.quote(cmd_str)}', + 'EB_SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )', f'echo Shell for the command: {shlex.quote(cmd_str)}', 'echo Use command history, exit to stop', - 'export PS1="eb-shell> $PS1"', - 'bash --norc', + 'bash --rcfile <(cat $EB_SCRIPT_DIR/env.sh; ' + 'echo \'PS1="eb-shell> "\'; ' + 'echo history -s {shlex.quote(cmd_str)})', ])) - os.chmod(filename, 0o775) + os.chmod(cmd_fp, 0o775) def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): @@ -354,9 +361,8 @@ def to_cmd_str(cmd): os.makedirs(toptmpdir, exist_ok=True) cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') - cmd_log_fp = os.path.join(tmpdir, 'cmd.sh') - _log.info(f'run_shell_cmd: command environment of "{cmd_str}" will be saved to {cmd_log_fp}') - save_cmd(cmd_str, work_dir, env, cmd_log_fp) + _log.info(f'run_shell_cmd: command environment of "{cmd_str}" will be saved to {tmpdir}') + save_cmd(cmd_str, work_dir, env, tmpdir) cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_shell_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') if split_stderr: From 9777222d1c7e158b8be1e2b24db47162a93f25d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 7 Apr 2024 21:48:18 +0200 Subject: [PATCH 223/430] Fix missing f-string --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 055bddfe5a..6f9b6e35a8 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -226,7 +226,7 @@ def save_cmd(cmd_str, work_dir, env, tmpdir): 'echo Use command history, exit to stop', 'bash --rcfile <(cat $EB_SCRIPT_DIR/env.sh; ' 'echo \'PS1="eb-shell> "\'; ' - 'echo history -s {shlex.quote(cmd_str)})', + f'echo history -s {shlex.quote(cmd_str)})', ])) os.chmod(cmd_fp, 0o775) From 3c20328835624c25711a727c1ab9441fbb90ad9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 7 Apr 2024 21:56:51 +0200 Subject: [PATCH 224/430] Add helpful comment to cmd.sh script --- easybuild/tools/run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 6f9b6e35a8..60d1d789be 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -219,6 +219,7 @@ def save_cmd(cmd_str, work_dir, env, tmpdir): cmd_fp = os.path.join(tmpdir, 'cmd.sh') with open(cmd_fp, 'w') as fid: fid.write('#!/usr/bin/env bash\n') + fid.write('# Run this script to replicate the environment that EB used to run the shell command\n') fid.write('\n'.join([ f'\ncd "{work_dir}"', 'EB_SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )', From e6ccf9e3f37cc1a4727976ca8b3eb12330d33216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Thu, 11 Apr 2024 18:07:55 +0000 Subject: [PATCH 225/430] Move PS1 and history to env file (simplifies and fixes escaping) --- easybuild/tools/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 60d1d789be..d0549a5c6b 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -214,6 +214,8 @@ def save_cmd(cmd_str, work_dir, env, tmpdir): # excludes bash functions (environment variables ending with %) fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in full_env.items() if not key.endswith('%'))) + fid.write('\n\nPS1="eb-shell> "') + fid.write(f'\nhistory -s {shlex.quote(cmd_str)}') # Make script that sets up bash shell with given environments set. cmd_fp = os.path.join(tmpdir, 'cmd.sh') @@ -225,9 +227,7 @@ def save_cmd(cmd_str, work_dir, env, tmpdir): 'EB_SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )', f'echo Shell for the command: {shlex.quote(cmd_str)}', 'echo Use command history, exit to stop', - 'bash --rcfile <(cat $EB_SCRIPT_DIR/env.sh; ' - 'echo \'PS1="eb-shell> "\'; ' - f'echo history -s {shlex.quote(cmd_str)})', + 'bash --rcfile $EB_SCRIPT_DIR/env.sh', ])) os.chmod(cmd_fp, 0o775) From 5ae9be778e6a8cc5266c15c818c71a578e1d2f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Mon, 15 Apr 2024 00:02:49 +0200 Subject: [PATCH 226/430] Changed trace output to mention directory --- easybuild/tools/run.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index d0549a5c6b..2069979ae4 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -372,7 +372,7 @@ def to_cmd_str(cmd): else: cmd_err_fp = None else: - cmd_out_fp, cmd_err_fp = None, None + tmpdir, cmd_out_fp, cmd_err_fp = None, None, None interactive = bool(qa_patterns) interactive_msg = 'interactive ' if interactive else '' @@ -390,7 +390,7 @@ def to_cmd_str(cmd): start_time = datetime.now() if not hidden: - _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, interactive=interactive) + _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, tmpdir, thread_id, interactive=interactive) if stream_output: print_msg(f"(streaming) output for command '{cmd_str}':") @@ -551,7 +551,7 @@ def to_cmd_str(cmd): return res -def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, interactive=False): +def _cmd_trace_msg(cmd, start_time, work_dir, stdin, tmpdir, thread_id, interactive=False): """ Helper function to construct and print trace message for command being run @@ -559,8 +559,7 @@ def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thr :param start_time: datetime object indicating when command was started :param work_dir: path of working directory in which command is run :param stdin: stdin input value for command - :param cmd_out_fp: path to output file for command - :param cmd_err_fp: path to errors/warnings output file for command + :param tmpdir: path to temporary output directory for command :param thread_id: thread ID (None when not running shell command asynchronously) :param interactive: boolean indicating whether it is an interactive command, or not """ @@ -580,10 +579,8 @@ def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thr ] if stdin: lines.append(f"\t[input: {stdin}]") - if cmd_out_fp: - lines.append(f"\t[output saved to {cmd_out_fp}]") - if cmd_err_fp: - lines.append(f"\t[errors/warnings saved to {cmd_err_fp}]") + if tmpdir: + lines.append(f"\t[output and state saved to {tmpdir}]") trace_msg('\n'.join(lines)) From c385640673169758c7da667292c36184c454f3aa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 15 Apr 2024 08:57:51 +0200 Subject: [PATCH 227/430] replace spaces/line breaks in question/wait patterns with regex pattern that matches one or more spaces/line breaks in run_shell_cmd --- easybuild/tools/run.py | 13 +++++++++---- test/framework/run.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 21eac92a70..702b624e5a 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -209,10 +209,14 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): """ match_found = False + space_line_break_pattern = r'[\s\n]+' + space_line_break_regex = re.compile(space_line_break_pattern) + stdout_end = stdout.decode(errors='ignore')[-1000:] for question, answers in qa_patterns: - # allow extra whitespace at the end - question += r'[\s\n]*$' + # replace spaces/line breaks with regex pattern that matches one or more spaces/line breaks, + # and allow extra whitespace at the end + question = space_line_break_pattern.join(space_line_break_regex.split(question)) + r'[\s\n]*$' regex = re.compile(question.encode()) res = regex.search(stdout) if res: @@ -245,8 +249,9 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): # if no match was found among question patterns, # take into account patterns for non-questions (qa_wait_patterns) for pattern in qa_wait_patterns: - # allow extra whitespace at the end - pattern += r'[\s\n]*$' + # replace spaces/line breaks with regex pattern that matches one or more spaces/line breaks, + # and allow extra whitespace at the end + pattern = space_line_break_pattern.join(space_line_break_regex.split(pattern)) + r'[\s\n]*$' regex = re.compile(pattern.encode()) if regex.search(stdout): _log.info(f"Found match for wait pattern '{pattern}'") diff --git a/test/framework/run.py b/test/framework/run.py index f1ee6955ec..8b2bc21fa0 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -871,6 +871,27 @@ def test_run_shell_cmd_qa(self): self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, "not-a-question-but-a-statement\nquestion\nanswer\n") + # test multi-line question + cmd = ';'.join([ + "echo please", + "echo answer", + "read x", + "echo $x", + ]) + qa = [("please answer", "42")] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "please\nanswer\n42\n") + + # also test multi-line wait pattern + cmd = "echo just; echo wait; sleep 3; " + cmd + qa_wait_patterns = ["just wait"] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=qa_wait_patterns, qa_timeout=1) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "just\nwait\nplease\nanswer\n42\n") + def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" From 804d5e281f419ef7df320154e7c2b1a517aed2d7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Apr 2024 09:31:32 +0200 Subject: [PATCH 228/430] remove unused import from test/framework/config.py --- test/framework/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/config.py b/test/framework/config.py index a6bd5a1233..337c7b9b87 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -38,7 +38,6 @@ from unittest import TextTestRunner import easybuild.tools.options as eboptions -from easybuild.tools import run from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import ERROR, IGNORE, WARN, BuildOptions, ConfigurationVariables from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath From c7199b6b512c9e9e750a77cc81fce0bba08515d6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Apr 2024 15:21:17 +0200 Subject: [PATCH 229/430] deprecate all functions that were moved from easybuild.tools.run to easybuild._deprecated --- easybuild/_deprecated.py | 62 ++++++++---- test/framework/run.py | 200 ++++++++++++++++++++++++--------------- 2 files changed, 169 insertions(+), 93 deletions(-) diff --git a/easybuild/_deprecated.py b/easybuild/_deprecated.py index f22ad0ba53..69920b9318 100644 --- a/easybuild/_deprecated.py +++ b/easybuild/_deprecated.py @@ -96,7 +96,7 @@ def cache_aware_func(cmd, *args, **kwargs): return cache_aware_func -def get_output_from_process(proc, read_size=None, asynchronous=False): +def get_output_from_process(proc, read_size=None, asynchronous=False, print_deprecation_warning=True): """ Get output from running process (that was opened with subprocess.Popen). @@ -105,6 +105,9 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): :param asynchronous: get output asynchronously """ + if print_deprecation_warning: + _log.deprecated("get_output_from_process is deprecated, you should stop using it", '6.0') + if asynchronous: # e=False is set to avoid raising an exception when command has completed; # that's needed to ensure we get all output, @@ -256,7 +259,8 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True return (proc, cmd, cwd, start_time, cmd_log) else: return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple, - regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks) + regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks, + print_deprecation_warning=False) def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''): @@ -275,11 +279,13 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out :result: dict value with result of the check (boolean 'done', 'exit_code', 'output') """ + _log.deprecated("check_async_cmd is deprecated, you should stop using it", '6.0') + # use small read size, to avoid waiting for a long time until sufficient output is produced if output_read_size: if not isinstance(output_read_size, int) or output_read_size < 0: raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)") - add_out = get_output_from_process(proc, read_size=output_read_size) + add_out = get_output_from_process(proc, read_size=output_read_size, print_deprecation_warning=False) _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out)) output += add_out @@ -290,7 +296,8 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out else: _log.debug("Asynchronous command '%s' completed!", cmd) output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, - simple=False, trace=False, log_ok=fail_on_error) + simple=False, trace=False, log_ok=fail_on_error, + print_deprecation_warning=False) done = True res = { @@ -302,7 +309,8 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, - regexp=True, stream_output=None, trace=True, output='', with_hook=True): + regexp=True, stream_output=None, trace=True, output='', with_hook=True, + print_deprecation_warning=True): """ Complete running of command represented by passed subprocess.Popen instance. @@ -319,6 +327,10 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False :param trace: print command being executed as part of trace output :param with_hook: trigger post run_shell_cmd hooks (if defined) """ + + if print_deprecation_warning: + _log.deprecated("complete_cmd is deprecated, you should stop using it", '6.0') + # use small read size when streaming output, to make it stream more fluently # read size should not be too small though, to avoid too much overhead if stream_output: @@ -333,7 +345,7 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False while ec is None: # need to read from time to time. # - otherwise the stdout/stderr buffer gets filled and it all stops working - output = get_output_from_process(proc, read_size=read_size) + output = get_output_from_process(proc, read_size=read_size, print_deprecation_warning=False) if cmd_log: cmd_log.write(output) if stream_output: @@ -342,7 +354,7 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False ec = proc.poll() # read remaining data (all of it) - output = get_output_from_process(proc) + output = get_output_from_process(proc, print_deprecation_warning=False) finally: proc.stdout.close() @@ -370,7 +382,7 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False except OSError as err: raise EasyBuildError("Failed to return to %s after executing command: %s", owd, err) - return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp) + return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp, print_deprecation_warning=False) def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, @@ -389,6 +401,9 @@ def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, re :param maxhits: maximum number of cycles (seconds) without being able to find a known question :param trace: print command being executed as part of trace output """ + + _log.deprecated("run_cmd_qa is deprecated, use run_shell_cmd from easybuild.tools.run instead", '6.0') + cwd = os.getcwd() if not isinstance(cmd, str) and len(cmd) > 1: @@ -549,7 +564,7 @@ def get_proc(): # need to read from time to time. # - otherwise the stdout/stderr buffer gets filled and it all stops working try: - out = get_output_from_process(proc, asynchronous=True) + out = get_output_from_process(proc, asynchronous=True, print_deprecation_warning=False) if cmd_log: cmd_log.write(out) @@ -622,7 +637,7 @@ def get_proc(): # Process stopped. Read all remaining data try: if proc.stdout: - out = get_output_from_process(proc) + out = get_output_from_process(proc, print_deprecation_warning=False) stdout_err += out if cmd_log: cmd_log.write(out) @@ -644,10 +659,10 @@ def get_proc(): except OSError as err: raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) - return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp) + return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp, print_deprecation_warning=False) -def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): +def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp, print_deprecation_warning=True): """ Parse command output and construct return value. :param cmd: executed command @@ -658,6 +673,10 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) """ + + if print_deprecation_warning: + _log.deprecated("parse_cmd_output is deprecated, you should stop using it", '6.0') + if strictness == IGNORE: check_ec = False fail_on_error_match = False @@ -688,7 +707,7 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in if fail_on_error_match or regexp: - res = parse_log_for_error(stdouterr, regexp, stdout=False) + res = parse_log_for_error(stdouterr, regexp, stdout=False, print_deprecation_warning=False) if res: errors = "\n\t" + "\n\t".join([r[0] for r in res]) error_str = "error" if len(res) == 1 else "errors" @@ -709,13 +728,17 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): return (stdouterr, ec) -def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): +def parse_log_for_error(txt, regExp=None, stdout=True, msg=None, print_deprecation_warning=True): """ txt is multiline string. - in memory regExp is a one-line regular expression - default """ + + if print_deprecation_warning: + _log.deprecated("parse_log_for_error is deprecated, you should stop using it", '6.0') + global errors_found_in_log if regExp and isinstance(regExp, bool): @@ -744,7 +767,7 @@ def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): return res -def extract_errors_from_log(log_txt, reg_exps): +def extract_errors_from_log(log_txt, reg_exps, print_deprecation_warning=True): """ Check provided string (command output) for messages matching specified regular expressions, and return 2-tuple with list of warnings and errors. @@ -753,6 +776,10 @@ def extract_errors_from_log(log_txt, reg_exps): or tuple of regular expression and action (any of [IGNORE, WARN, ERROR]) :return: (warnings, errors) as lists of lines containing a match """ + + if print_deprecation_warning: + _log.deprecated("extract_errors_from_log is deprecated, you should stop using it", '6.0') + actions = (IGNORE, WARN, ERROR) # promote single string value to list, since code below expects a list @@ -799,8 +826,11 @@ def check_log_for_errors(log_txt, reg_exps): :param reg_exps: List of: regular expressions (as strings) to error on, or tuple of regular expression and action (any of [IGNORE, WARN, ERROR]) """ + + _log.deprecated("check_log_for_errors is deprecated, you should stop using it", '6.0') + global errors_found_in_log - warnings, errors = extract_errors_from_log(log_txt, reg_exps) + warnings, errors = extract_errors_from_log(log_txt, reg_exps, print_deprecation_warning=False) errors_found_in_log += len(warnings) + len(errors) if warnings: diff --git a/test/framework/run.py b/test/framework/run.py index 2e0837e225..308c086f94 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -76,6 +76,9 @@ def tearDown(self): def test_get_output_from_process(self): """Test for get_output_from_process utility function.""" + # use of get_output_from_process is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + @contextlib.contextmanager def get_proc(cmd, asynchronous=False): if asynchronous: @@ -92,53 +95,58 @@ def get_proc(cmd, asynchronous=False): subprocess_terminate(proc, timeout=1) # get all output at once - with get_proc("echo hello") as proc: - out = get_output_from_process(proc) - self.assertEqual(out, 'hello\n') + with self.mocked_stdout_stderr(): + with get_proc("echo hello") as proc: + out = get_output_from_process(proc) + self.assertEqual(out, 'hello\n') # first get 100 bytes, then get the rest all at once - with get_proc("echo hello") as proc: - out = get_output_from_process(proc, read_size=100) - self.assertEqual(out, 'hello\n') - out = get_output_from_process(proc) - self.assertEqual(out, '') + with self.mocked_stdout_stderr(): + with get_proc("echo hello") as proc: + out = get_output_from_process(proc, read_size=100) + self.assertEqual(out, 'hello\n') + out = get_output_from_process(proc) + self.assertEqual(out, '') # get output in small bits, keep trying to get output (which shouldn't fail) - with get_proc("echo hello") as proc: - out = get_output_from_process(proc, read_size=1) - self.assertEqual(out, 'h') - out = get_output_from_process(proc, read_size=3) - self.assertEqual(out, 'ell') - out = get_output_from_process(proc, read_size=2) - self.assertEqual(out, 'o\n') - out = get_output_from_process(proc, read_size=1) - self.assertEqual(out, '') - out = get_output_from_process(proc, read_size=10) - self.assertEqual(out, '') - out = get_output_from_process(proc) - self.assertEqual(out, '') + with self.mocked_stdout_stderr(): + with get_proc("echo hello") as proc: + out = get_output_from_process(proc, read_size=1) + self.assertEqual(out, 'h') + out = get_output_from_process(proc, read_size=3) + self.assertEqual(out, 'ell') + out = get_output_from_process(proc, read_size=2) + self.assertEqual(out, 'o\n') + out = get_output_from_process(proc, read_size=1) + self.assertEqual(out, '') + out = get_output_from_process(proc, read_size=10) + self.assertEqual(out, '') + out = get_output_from_process(proc) + self.assertEqual(out, '') # can also get output asynchronously (read_size is *ignored* in that case) async_cmd = "echo hello; read reply; echo $reply" - with get_proc(async_cmd, asynchronous=True) as proc: - out = get_output_from_process(proc, asynchronous=True) - self.assertEqual(out, 'hello\n') - asyncprocess.send_all(proc, 'test123\n') - out = get_output_from_process(proc) - self.assertEqual(out, 'test123\n') + with self.mocked_stdout_stderr(): + with get_proc(async_cmd, asynchronous=True) as proc: + out = get_output_from_process(proc, asynchronous=True) + self.assertEqual(out, 'hello\n') + asyncprocess.send_all(proc, 'test123\n') + out = get_output_from_process(proc) + self.assertEqual(out, 'test123\n') - with get_proc(async_cmd, asynchronous=True) as proc: - out = get_output_from_process(proc, asynchronous=True, read_size=1) - # read_size is ignored when getting output asynchronously, we're getting more than 1 byte! - self.assertEqual(out, 'hello\n') - asyncprocess.send_all(proc, 'test123\n') - out = get_output_from_process(proc, read_size=3) - self.assertEqual(out, 'tes') - out = get_output_from_process(proc, read_size=2) - self.assertEqual(out, 't1') - out = get_output_from_process(proc) - self.assertEqual(out, '23\n') + with self.mocked_stdout_stderr(): + with get_proc(async_cmd, asynchronous=True) as proc: + out = get_output_from_process(proc, asynchronous=True, read_size=1) + # read_size is ignored when getting output asynchronously, we're getting more than 1 byte! + self.assertEqual(out, 'hello\n') + asyncprocess.send_all(proc, 'test123\n') + out = get_output_from_process(proc, read_size=3) + self.assertEqual(out, 'tes') + out = get_output_from_process(proc, read_size=2) + self.assertEqual(out, 't1') + out = get_output_from_process(proc) + self.assertEqual(out, '23\n') def test_run_cmd(self): """Basic test for run_cmd function.""" @@ -309,7 +317,7 @@ def test_run_shell_cmd_log(self): def test_run_cmd_negative_exit_code(self): """Test run_cmd function with command that has negative exit code.""" - # use of run_cmd is deprecated, so we need to allow it here + # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here self.allow_deprecated_behaviour() # define signal handler to call in case run_cmd takes too long @@ -777,6 +785,9 @@ def test_run_shell_cmd_trace_stdin(self): def test_run_cmd_qa(self): """Basic test for run_cmd_qa function.""" + # use of run_cmd_qa is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + cmd = "echo question; read x; echo $x" qa = {'question': 'answer'} with self.mocked_stdout_stderr(): @@ -812,7 +823,8 @@ def test_run_cmd_qa(self): error_pattern = "Max nohits 1 reached: end of output not-a-question-but-a-statement" self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd_qa, cmd, qa, maxhits=1, trace=False) - (out, ec) = run_cmd_qa(cmd, qa, no_qa=["not-a-question-but-a-statement"], maxhits=1, trace=False) + with self.mocked_stdout_stderr(): + (out, ec) = run_cmd_qa(cmd, qa, no_qa=["not-a-question-but-a-statement"], maxhits=1, trace=False) self.assertEqual(out, "not-a-question-but-a-statement\nquestion\nanswer\n") self.assertEqual(ec, 0) @@ -901,6 +913,9 @@ def test_run_shell_cmd_qa(self): def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" + # use of run_cmd_qa is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + # command that generates a lot of output before waiting for input # note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000! cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; ' @@ -964,6 +979,10 @@ def test_run_shell_cmd_qa_buffering(self): def test_run_cmd_qa_log_all(self): """Test run_cmd_qa with log_output enabled""" + + # use of run_cmd_qa is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + with self.mocked_stdout_stderr(): (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}, log_all=True) self.assertEqual(ec, 0) @@ -990,7 +1009,7 @@ def test_run_shell_cmd_qa_log(self): def test_run_cmd_qa_trace(self): """Test run_cmd under --trace""" - # use of run_cmd is deprecated, so we need to allow it here + # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here self.allow_deprecated_behaviour() # --trace is enabled by default @@ -1001,7 +1020,7 @@ def test_run_cmd_qa_trace(self): stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) - self.assertEqual(stderr, '') + self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality")) pattern = r"^ >> running interactive command:\n" pattern += r"\t\[started at: .*\]\n" pattern += r"\t\[working dir: .*\]\n" @@ -1054,6 +1073,10 @@ def test_run_shell_cmd_qa_trace(self): def test_run_cmd_qa_answers(self): """Test providing list of answers in run_cmd_qa.""" + + # use of run_cmd_qa is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + cmd = "echo question; read x; echo $x; " * 2 qa = {"question": ["answer1", "answer2"]} @@ -1197,7 +1220,12 @@ def test_run_shell_cmd_cache(self): def test_parse_log_error(self): """Test basic parse_log_for_error functionality.""" - errors = parse_log_for_error("error failed", True) + + # use of parse_log_for_error is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + + with self.mocked_stdout_stderr(): + errors = parse_log_for_error("error failed", True) self.assertEqual(len(errors), 1) def test_run_cmd_dry_run(self): @@ -1419,7 +1447,7 @@ def test_run_shell_cmd_stream(self): def test_run_cmd_async(self): """Test asynchronously running of a shell command via run_cmd + complete_cmd.""" - # use of run_cmd is deprecated, so we need to allow it here + # use of run_cmd/check_async_cmd/get_output_from_process is deprecated, so we need to allow it here self.allow_deprecated_behaviour() os.environ['TEST'] = 'test123' @@ -1453,13 +1481,15 @@ def test_run_cmd_async(self): # first check, only read first 12 output characters # (otherwise we'll be waiting until command is completed) - res = check_async_cmd(*cmd_info, output_read_size=12) + with self.mocked_stdout_stderr(): + res = check_async_cmd(*cmd_info, output_read_size=12) self.assertEqual(res, {'done': False, 'exit_code': None, 'output': 'sleeping...\n'}) # 2nd check with default output size (1024) gets full output # (keep checking until command is fully done) - while not res['done']: - res = check_async_cmd(*cmd_info, output=res['output']) + with self.mocked_stdout_stderr(): + while not res['done']: + res = check_async_cmd(*cmd_info, output=res['output']) self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'}) # check asynchronous running of failing command @@ -1468,14 +1498,16 @@ def test_run_cmd_async(self): cmd_info = run_cmd(error_test_cmd, asynchronous=True) time.sleep(1) error_pattern = 'cmd ".*" exited with exit code 123' - self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info) + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info) with self.mocked_stdout_stderr(): cmd_info = run_cmd(error_test_cmd, asynchronous=True) - res = check_async_cmd(*cmd_info, fail_on_error=False) + res = check_async_cmd(*cmd_info, fail_on_error=False) # keep checking until command is fully done - while not res['done']: - res = check_async_cmd(*cmd_info, fail_on_error=False, output=res['output']) + with self.mocked_stdout_stderr(): + while not res['done']: + res = check_async_cmd(*cmd_info, fail_on_error=False, output=res['output']) self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"}) # also test with a command that produces a lot of output, @@ -1498,10 +1530,11 @@ def test_run_cmd_async(self): ec = proc.poll() self.assertEqual(ec, None) - while ec is None: - time.sleep(1) - output += get_output_from_process(proc) - ec = proc.poll() + with self.mocked_stdout_stderr(): + while ec is None: + time.sleep(1) + output += get_output_from_process(proc) + ec = proc.poll() with self.mocked_stdout_stderr(): out, ec = complete_cmd(*cmd_info, simple=False, output=output) @@ -1514,8 +1547,9 @@ def test_run_cmd_async(self): cmd_info = run_cmd(verbose_test_cmd, asynchronous=True) error_pattern = r"Number of output bytes to read should be a positive integer value \(or zero\)" - self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1) - self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo') + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1) + self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo') # with output_read_size set to 0, no output is read yet, only status of command is checked with self.mocked_stdout_stderr(): @@ -1531,8 +1565,9 @@ def test_run_cmd_async(self): self.assertTrue(res['output'].startswith('start\n')) self.assertFalse(res['output'].endswith('\ndone\n')) # keep checking until command is complete - while not res['done']: - res = check_async_cmd(*cmd_info, output=res['output']) + with self.mocked_stdout_stderr(): + while not res['done']: + res = check_async_cmd(*cmd_info, output=res['output']) self.assertEqual(res['done'], True) self.assertEqual(res['exit_code'], 0) self.assertEqual(len(res['output']), 435661) @@ -1599,13 +1634,19 @@ def test_run_shell_cmd_async(self): self.assertTrue(res.output.endswith('\nfoo501000\ndone\n')) def test_check_log_for_errors(self): + """Test for check_log_for_errors""" + + # use of check_log_for_errors is deprecated, so we need to allow it here + self.allow_deprecated_behaviour() + fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) - self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [42]) - self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [(42, IGNORE)]) - self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", "invalid-mode")]) - self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", IGNORE, "")]) + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [42]) + self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [(42, IGNORE)]) + self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", "invalid-mode")]) + self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", IGNORE, "")]) input_text = "\n".join([ "OK", @@ -1620,20 +1661,24 @@ def test_check_log_for_errors(self): r"\tthe process crashed with 0" # String promoted to list - self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, - r"\b(error|crashed)\b") + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, + r"\b(error|crashed)\b") # List of string(s) - self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, - [r"\b(error|crashed)\b"]) + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, + [r"\b(error|crashed)\b"]) # List of tuple(s) - self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, - [(r"\b(error|crashed)\b", ERROR)]) + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, + [(r"\b(error|crashed)\b", ERROR)]) expected_msg = "Found 2 potential error(s) in command output:\n"\ "\terror found\n"\ "\tthe process crashed with 0" init_logging(logfile, silent=True) - check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)]) + with self.mocked_stdout_stderr(): + check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)]) stop_logging(logfile) self.assertIn(expected_msg, read_file(logfile)) @@ -1642,12 +1687,13 @@ def test_check_log_for_errors(self): r"\ttest failed" write_file(logfile, '') init_logging(logfile, silent=True) - self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [ - r"\berror\b", - (r"\ballowed-test failed\b", IGNORE), - (r"(?i)\bCRASHED\b", WARN), - "fail" - ]) + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [ + r"\berror\b", + (r"\ballowed-test failed\b", IGNORE), + (r"(?i)\bCRASHED\b", WARN), + "fail" + ]) stop_logging(logfile) expected_msg = "Found 1 potential error(s) in command output:\n\tthe process crashed with 0" self.assertIn(expected_msg, read_file(logfile)) @@ -1657,7 +1703,7 @@ def test_run_cmd_with_hooks(self): Test running command with run_cmd with pre/post run_shell_cmd hooks in place. """ - # use of run_cmd is deprecated, so we need to allow it here + # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here self.allow_deprecated_behaviour() cwd = os.getcwd() From 2564f76dbb0ef6bbcd6f5e239981e6c65451e002 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Apr 2024 17:25:22 +0200 Subject: [PATCH 230/430] suppress deprecation warning from run_cmd_qa in test_run_cmd_qa --- test/framework/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index 308c086f94..b596319c8e 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -821,7 +821,8 @@ def test_run_cmd_qa(self): # fails because non-question is encountered error_pattern = "Max nohits 1 reached: end of output not-a-question-but-a-statement" - self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd_qa, cmd, qa, maxhits=1, trace=False) + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd_qa, cmd, qa, maxhits=1, trace=False) with self.mocked_stdout_stderr(): (out, ec) = run_cmd_qa(cmd, qa, no_qa=["not-a-question-but-a-statement"], maxhits=1, trace=False) From 43fb353134b3fa4ed1f7b4443b7854823eacf63d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 17 Apr 2024 09:07:51 +0200 Subject: [PATCH 231/430] take into account that question patterns may include a hard space --- easybuild/tools/run.py | 4 ++++ test/framework/run.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 702b624e5a..17c3361a0d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -214,6 +214,8 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): stdout_end = stdout.decode(errors='ignore')[-1000:] for question, answers in qa_patterns: + # first replace hard spaces by regular spaces, since they would mess up the join/split below + question = question.replace(r'\ ', ' ') # replace spaces/line breaks with regex pattern that matches one or more spaces/line breaks, # and allow extra whitespace at the end question = space_line_break_pattern.join(space_line_break_regex.split(question)) + r'[\s\n]*$' @@ -249,6 +251,8 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): # if no match was found among question patterns, # take into account patterns for non-questions (qa_wait_patterns) for pattern in qa_wait_patterns: + # first replace hard spaces by regular spaces, since they would mess up the join/split below + pattern = pattern.replace(r'\ ', ' ') # replace spaces/line breaks with regex pattern that matches one or more spaces/line breaks, # and allow extra whitespace at the end pattern = space_line_break_pattern.join(space_line_break_regex.split(pattern)) + r'[\s\n]*$' diff --git a/test/framework/run.py b/test/framework/run.py index 8b2bc21fa0..8e5f790327 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -892,6 +892,20 @@ def test_run_shell_cmd_qa(self): self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, "just\nwait\nplease\nanswer\n42\n") + # test multi-line question pattern with hard space + cmd = ';'.join([ + "echo please", + "echo answer", + "read x", + "echo $x", + ]) + # question pattern uses hard space, should get replaced internally by more liberal whitespace regex pattern + qa = [(r"please\ answer", "42")] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa, qa_timeout=3) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "please\nanswer\n42\n") + def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" From f0bc1a48642fb98dcdd4ab0548e557a2af70e9ba Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Wed, 17 Apr 2024 15:06:38 +0100 Subject: [PATCH 232/430] add support for alternate easyconfig parameters --- easybuild/framework/easyconfig/easyconfig.py | 6 ++++-- easybuild/framework/easyconfig/parser.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 5f8170b748..ab816dc8f8 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -59,7 +59,7 @@ from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION, retrieve_blocks_in_spec from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS +from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools import LooseVersion @@ -118,6 +118,8 @@ def handle_deprecated_or_replaced_easyconfig_parameters(ec_method): def new_ec_method(self, key, *args, **kwargs): """Check whether any replace easyconfig parameters are still used""" # map deprecated parameters to their replacements, issue deprecation warning(/error) + if key in ALTERNATE_PARAMETERS: + key = ALTERNATE_PARAMETERS[key] if key in DEPRECATED_PARAMETERS: depr_key = key key, ver = DEPRECATED_PARAMETERS[depr_key] @@ -179,7 +181,7 @@ def triage_easyconfig_params(variables, ec): for key in variables: # validations are skipped, just set in the config - if key in ec or key in DEPRECATED_PARAMETERS.keys(): + if any(key in d for d in (ec, DEPRECATED_PARAMETERS.keys(), ALTERNATE_PARAMETERS.keys()): ec_params[key] = variables[key] _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key])) elif key in REPLACED_PARAMETERS: diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index 8f08500815..10912f0054 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -42,6 +42,11 @@ from easybuild.tools.filetools import read_file, write_file +# alternate easyconfig parameters, and their non-deprecated equivalents +ALTNERATE_PARAMETERS = { + # : , +} + # deprecated easyconfig parameters, and their replacements DEPRECATED_PARAMETERS = { # : (, ), From b86e0fdce8d9dfb8905090deb7f9b410c0ef4943 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Mon, 18 Mar 2024 16:59:32 +0000 Subject: [PATCH 233/430] add mechanism for easyconfig template deprecation --- easybuild/framework/easyconfig/easyconfig.py | 21 ++++++++++++++++++-- easybuild/framework/easyconfig/templates.py | 10 ++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ab816dc8f8..76970acb56 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -61,7 +61,8 @@ from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict +from easybuild.framework.easyconfig.templates import DEPRECATED_TEMPLATES, TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC +from easybuild.framework.easyconfig.templates import template_constant_dict from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG @@ -1996,7 +1997,23 @@ def resolve_template(value, tmpl_dict): try: value = value % tmpl_dict except KeyError: - _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict) + # check if any deprecated templates resolve + try: + orig_value = value + # map old templates to new values + depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in + DEPRECATED_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} + value = value % depr_map + + for old_tmpl, val in depr_map.items(): + # check which deprecated templates were replaced, and issue deprecation warnings + if old_tmpl in orig_value and val in value: + new_tmpl, ver = DEPRECATED_TEMPLATES[old_tmpl] + _log.deprecated("Easyconfig template '%s' is deprecated, use '%s' instead" + % (old_tmpl, new_tmpl), ver) + except KeyError: + _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict) + else: # this block deals with references to objects and returns other references # for reading this is ok, but for self['x'] = {} diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 388d80f97a..79215a7609 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -161,6 +161,16 @@ ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'), ] +# deprecated templates, and their replacements +DEPRECATED_TEMPLATES = { + # : (, ), +} + +# deprecated template constants, and their replacements +DEPRECATED_TEMPLATE_CONSTANTS = { + # : (, ), +} + extensions = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z'] for ext in extensions: suffix = ext.replace('.', '_').upper() From 9b4c67402d13360050fde76cd96b4913d8f4541f Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Wed, 27 Mar 2024 15:10:14 +0000 Subject: [PATCH 234/430] handle deprecated template constants --- .../easyconfig/format/pyheaderconfigobj.py | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 8227a40958..eb5e77d5ea 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -39,7 +39,7 @@ from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS +from easybuild.framework.easyconfig.templates import DEPRECATED_TEMPLATE_CONSTANTS, TEMPLATE_CONSTANTS from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import ConfigObj from easybuild.tools.systemtools import get_shared_lib_ext @@ -86,6 +86,55 @@ def build_easyconfig_variables_dict(): return vars_dict +def handle_deprecated_constants(method): + """Decorator to handle deprecated easyconfig template constants""" + def wrapper(self, key, *args, **kwargs): + """Check whether any deprecated constants are used""" + deprecated = DEPRECATED_TEMPLATE_CONSTANTS + if key in deprecated: + depr_key = key + key, ver = deprecated[depr_key] + _log.deprecated(f"Easyconfig template constant '{depr_key}' is deprecated, use '{key}' instead", ver) + return method(self, key, *args, **kwargs) + return wrapper + + +class DeprecatedDict(dict): + """Custom dictionary that handles deprecated easyconfig template constants gracefully""" + + def __init__(self, *args, **kwargs): + self.clear() + self.update(*args, **kwargs) + + @handle_deprecated_constants + def __contains__(self, key): + return super().__contains__(key) + + @handle_deprecated_constants + def __delitem__(self, key): + return super().__delitem__(key) + + @handle_deprecated_constants + def __getitem__(self, key): + return super().__getitem__(key) + + @handle_deprecated_constants + def __setitem__(self, key, value): + return super().__setitem__(key, value) + + def update(self, *args, **kwargs): + if args: + if isinstance(args[0], dict): + for key, value in args[0].items(): + self.__setitem__(key, value) + else: + for key, value in args[0]: + self.__setitem__(key, value) + + for key, value in kwargs.items(): + self.__setitem__(key, value) + + class EasyConfigFormatConfigObj(EasyConfigFormat): """ Extended EasyConfig format, with support for a header and sections that are actually parsed (as opposed to exec'ed). @@ -176,7 +225,7 @@ def parse_header(self, header): def parse_pyheader(self, pyheader): """Parse the python header, assign to docstring and cfg""" - global_vars = self.pyheader_env() + global_vars = DeprecatedDict(self.pyheader_env()) self.log.debug("pyheader initial global_vars %s", global_vars) self.log.debug("pyheader text being exec'ed: %s", pyheader) From 005934439db84996d31477533eae151aa2b80f10 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Wed, 17 Apr 2024 15:25:15 +0100 Subject: [PATCH 235/430] add support for alternate templates and constants --- easybuild/framework/easyconfig/easyconfig.py | 13 ++++++++----- .../easyconfig/format/pyheaderconfigobj.py | 8 ++++++-- easybuild/framework/easyconfig/templates.py | 10 ++++++++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 76970acb56..153d95534c 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -61,8 +61,8 @@ from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.templates import DEPRECATED_TEMPLATES, TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC -from easybuild.framework.easyconfig.templates import template_constant_dict +from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATES, DEPRECATED_TEMPLATES, TEMPLATE_CONSTANTS, +from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG @@ -1997,13 +1997,16 @@ def resolve_template(value, tmpl_dict): try: value = value % tmpl_dict except KeyError: - # check if any deprecated templates resolve + # check if any alternate or deprecated templates resolve try: orig_value = value - # map old templates to new values + # map old templates to new values for alternate and deprecated templates + alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in + ALTERNATE_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in DEPRECATED_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} - value = value % depr_map + # try templating with tmpl_dict, alt_map and depr_map + value = value % {**tmpl_dict, **alt_map, **depr_map} for old_tmpl, val in depr_map.items(): # check which deprecated templates were replaced, and issue deprecation warnings diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index eb5e77d5ea..9d3189804a 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -39,7 +39,8 @@ from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.templates import DEPRECATED_TEMPLATE_CONSTANTS, TEMPLATE_CONSTANTS +from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATE_CONSTANTS, DEPRECATED_TEMPLATE_CONSTANTS +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import ConfigObj from easybuild.tools.systemtools import get_shared_lib_ext @@ -90,8 +91,11 @@ def handle_deprecated_constants(method): """Decorator to handle deprecated easyconfig template constants""" def wrapper(self, key, *args, **kwargs): """Check whether any deprecated constants are used""" + alternate = ALTERNATE_TEMPLATE_CONSTANTS deprecated = DEPRECATED_TEMPLATE_CONSTANTS - if key in deprecated: + if key in alternate: + key = alternate[key] + elif key in deprecated: depr_key = key key, ver = deprecated[depr_key] _log.deprecated(f"Easyconfig template constant '{depr_key}' is deprecated, use '{key}' instead", ver) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 79215a7609..00d995a4b9 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -161,11 +161,21 @@ ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'), ] +# alternate templates, and their equivalents +ALTERNATE_TEMPLATES = { + # : , +} + # deprecated templates, and their replacements DEPRECATED_TEMPLATES = { # : (, ), } +# alternate template constants, and their equivalents +ALTERNATE_TEMPLATE_CONSTANTS = { + # : , +} + # deprecated template constants, and their replacements DEPRECATED_TEMPLATE_CONSTANTS = { # : (, ), From a544d6b1138a0681268bdf61a53c52eed4437440 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Wed, 17 Apr 2024 15:28:11 +0100 Subject: [PATCH 236/430] use elif when checking if keys are alternate/deprecated/replaced ec params --- easybuild/framework/easyconfig/easyconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 153d95534c..279f06a7fe 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -121,11 +121,11 @@ def new_ec_method(self, key, *args, **kwargs): # map deprecated parameters to their replacements, issue deprecation warning(/error) if key in ALTERNATE_PARAMETERS: key = ALTERNATE_PARAMETERS[key] - if key in DEPRECATED_PARAMETERS: + elif key in DEPRECATED_PARAMETERS: depr_key = key key, ver = DEPRECATED_PARAMETERS[depr_key] _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver) - if key in REPLACED_PARAMETERS: + elif key in REPLACED_PARAMETERS: _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') return ec_method(self, key, *args, **kwargs) From 53b82fdd7010cd89ec2300a57f98e61be0be0c45 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Wed, 17 Apr 2024 15:32:55 +0100 Subject: [PATCH 237/430] fix missing bracket --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 279f06a7fe..077b0f4bf8 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -182,7 +182,7 @@ def triage_easyconfig_params(variables, ec): for key in variables: # validations are skipped, just set in the config - if any(key in d for d in (ec, DEPRECATED_PARAMETERS.keys(), ALTERNATE_PARAMETERS.keys()): + if any(key in d for d in (ec, DEPRECATED_PARAMETERS.keys(), ALTERNATE_PARAMETERS.keys())): ec_params[key] = variables[key] _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key])) elif key in REPLACED_PARAMETERS: From a671b7e13440ca1b66cbeac6e94f01372a56ebea Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Wed, 17 Apr 2024 15:33:58 +0100 Subject: [PATCH 238/430] remove trailing comma in imports --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 077b0f4bf8..2857006a20 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -61,7 +61,7 @@ from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATES, DEPRECATED_TEMPLATES, TEMPLATE_CONSTANTS, +from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATES, DEPRECATED_TEMPLATES, TEMPLATE_CONSTANTS from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg From f3401f603801fde2e937d3eb79ed26d871f01b51 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Wed, 17 Apr 2024 15:42:18 +0100 Subject: [PATCH 239/430] fix typo --- easybuild/framework/easyconfig/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index 10912f0054..0a177c089a 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -43,7 +43,7 @@ # alternate easyconfig parameters, and their non-deprecated equivalents -ALTNERATE_PARAMETERS = { +ALTERNATE_PARAMETERS = { # : , } From c01db56615ac82dd135beeb2cd41288134b144a5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 18 Apr 2024 10:32:46 +0200 Subject: [PATCH 240/430] Fix --dump-env-script with existing modules This option should only require `--force` when the env file exists. Otherwise, when the module already exists `--force` should not be required. In any case the module file must not be deleted. --- easybuild/framework/easyblock.py | 2 +- easybuild/main.py | 6 ++--- easybuild/tools/config.py | 1 + test/framework/options.py | 39 ++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b17806b7c2..33110c61dd 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2349,7 +2349,7 @@ def check_readiness_step(self): self.log.info("No module %s found. Not skipping anything." % self.full_mod_name) # remove existing module file under --force (but only if --skip is not used) - elif build_option('force') or build_option('rebuild'): + elif (build_option('force') or build_option('rebuild')) and not build_option('dump_env_script'): self.remove_module_file() def fetch_step(self, skip_checksums=False): diff --git a/easybuild/main.py b/easybuild/main.py index df5f23a460..42bd83dff5 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -440,9 +440,9 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules keep_available_modules = any(( - forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums, - options.sanity_check_only, options.inject_checksums_to_json) - ) + forced, dry_run_mode, any_pr_option_set, options.copy_ec, options.dump_env_script, options.extended_dry_run, + options.inject_checksums, options.inject_checksums_to_json, options.sanity_check_only + )) # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not keep_available_modules: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 3a5cbecb55..6bec64764c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -278,6 +278,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'debug', 'debug_lmod', 'dump_autopep8', + 'dump_env_script', 'enforce_checksums', 'experimental', 'extended_dry_run', diff --git a/test/framework/options.py b/test/framework/options.py index b2f3da43c1..29b3b3a4d6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5249,6 +5249,45 @@ def test_dump_env_script(self): ]) self.assertEqual(out.strip(), expected_out) + def test_dump_env_script_existing_module(self): + toy_ec = 'toy-0.0.eb' + + os.chdir(self.test_prefix) + self._run_mock_eb([toy_ec, '--force'], do_build=True) + env_script = os.path.join(self.test_prefix, os.path.splitext(toy_ec)[0] + '.env') + test_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + test_module += '.lua' + self.assertExists(test_module) + self.assertNotExists(env_script) + + args = [toy_ec, '--dump-env'] + os.chdir(self.test_prefix) + self._run_mock_eb(args, do_build=True, raise_error=True) + self.assertExists(env_script) + self.assertExists(test_module) + module_content = read_file(test_module) + env_file_content = read_file(env_script) + + error_msg = (r"Script\(s\) already exists, not overwriting them \(unless --force is used\): " + + os.path.basename(env_script)) + os.chdir(self.test_prefix) + self.assertErrorRegex(EasyBuildError, error_msg, self._run_mock_eb, args, do_build=True, raise_error=True) + self.assertExists(env_script) + self.assertExists(test_module) + # Unchanged module and env file + self.assertEqual(read_file(test_module), module_content) + self.assertEqual(read_file(env_script), env_file_content) + + args.append('--force') + os.chdir(self.test_prefix) + self._run_mock_eb(args, do_build=True, raise_error=True) + self.assertExists(env_script) + self.assertExists(test_module) + # Unchanged module and env file + self.assertEqual(read_file(test_module), module_content) + self.assertEqual(read_file(env_script), env_file_content) + def test_stop(self): """Test use of --stop.""" args = ['toy-0.0.eb', '--force', '--stop=configure'] From 45ad2280b76700362669c5726fb3c069de3bedc0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 18 Apr 2024 10:47:35 +0200 Subject: [PATCH 241/430] Remove some stray code --- test/framework/easyblock.py | 2 +- test/framework/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e6b54e0bc8..8ea7bf43fe 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1685,7 +1685,7 @@ def test_fetch_patches(self): self.assertEqual(eb.patches[1]['level'], 4) self.assertEqual(eb.patches[2]['name'], toy_patch) self.assertEqual(eb.patches[2]['sourcepath'], 'foobar') - self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'), + self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz') self.assertEqual(eb.patches[3]['copy'], 'some/path') self.assertEqual(eb.patches[4]['name'], toy_patch) self.assertEqual(eb.patches[4]['level'], 0) diff --git a/test/framework/options.py b/test/framework/options.py index 29b3b3a4d6..3deaf8cf9c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4522,7 +4522,7 @@ def test_github_new_update_pr(self): res = [d for d in res if os.path.basename(d) != os.path.basename(git_working_dir)] if len(res) == 1: unstaged_file_full = os.path.join(res[0], unstaged_file) - self.assertNotExists(unstaged_file_full), "%s not found in %s" % (unstaged_file, res[0]) + self.assertNotExists(unstaged_file_full) else: self.fail("Found copy of easybuild-easyconfigs working copy") From 668d044a8efafd474f861c2d3a040f887f7cc537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 21 Apr 2024 02:10:58 +0200 Subject: [PATCH 242/430] Fix tests for new run command output dir --- test/framework/run.py | 6 +++--- test/framework/toy_build.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 23fdb562d3..eb0f76c457 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -676,7 +676,7 @@ def test_run_shell_cmd_trace(self): r"\techo hello", r"\t\[started at: .*\]", r"\t\[working dir: .*\]", - r"\t\[output saved to .*\]", + r"\t\[output and state saved to .*\]", r" >> command completed: exit 0, ran in .*", ] @@ -736,7 +736,7 @@ def test_run_shell_cmd_trace_stdin(self): r"\techo hello", r"\t\[started at: [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\]", r"\t\[working dir: .*\]", - r"\t\[output saved to .*\]", + r"\t\[output and state saved to .*\]", r" >> command completed: exit 0, ran in .*", ] @@ -1092,7 +1092,7 @@ def test_run_shell_cmd_qa_trace(self): pattern += r"\techo \'n: \'; read n; seq 1 \$n\n" pattern += r"\t\[started at: .*\]\n" pattern += r"\t\[working dir: .*\]\n" - pattern += r"\t\[output saved to .*\]\n" + pattern += r"\t\[output and state saved to .*\]\n" pattern += r' >> command completed: exit 0, ran in .*' self.assertTrue(re.search(pattern, stdout), "Pattern '%s' found in: %s" % (pattern, stdout)) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 121bb43367..5a7d1654cf 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2985,7 +2985,7 @@ def test_toy_build_trace(self): r"\tgcc toy.c -o toy\n" r"\t\[started at: .*\]", r"\t\[working dir: .*\]", - r"\t\[output saved to .*\]", + r"\t\[output and state saved to .*\]", r'', ]), r" >> command completed: exit 0, ran in .*", From cf925132f32bb508fc713071ad97e3bfabe87b00 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Mon, 22 Apr 2024 05:46:35 +0100 Subject: [PATCH 243/430] add template deprecation test --- test/framework/easyconfig.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 7dc088bff3..892fbc6abf 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1451,6 +1451,36 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) + def test_template_deprecation(self): + """Test deprecation of templates""" + + template_test_deprecations = { + 'builddir': ('new_build_dir', '1000000000'), + 'cudaver': ('new_cuda_ver', '1000000000'), + 'start_dir': ('new_start_dir', '1000000000'), + } + easyconfig.templates.DEPRECATED_TEMPLATES.update(template_test_deprecations) + + tmpl_str = "cd %(start_dir)s && make PREFIX=%(installdir)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" + tmpl_dict = { + 'new_build_dir': '/example/build_dir', + 'new_cuda_ver': '12.1.1', + 'installdir': '/example/installdir', + 'new_start_dir': '/example/build_dir/start_dir', + } + + with self.mocked_stdout_stderr() as (_, stderr): + res = resolve_template(tmpl_str, tmpl_dict) + stderr = stderr.getvalue() + + for tmpl in ['builddir', 'cudaver', 'installdir', 'start_dir']: + self.assertNotIn("%(" + tmpl + ")s", res) + + for old, (new, ver) in template_test_deprecations.items(): + depr_str = (f"WARNING: Deprecated functionality, will no longer work in v{ver}: Easyconfig template '{old}'" + f" is deprecated, use '{new}' instead") + self.assertIn(depr_str, stderr) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() From 88eef0631f07f925474da5f24693145930e2203b Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Mon, 22 Apr 2024 05:58:03 +0100 Subject: [PATCH 244/430] enhance template deprecation test to also test alternate templates --- test/framework/easyconfig.py | 40 +++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 892fbc6abf..e34d8eb960 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -120,6 +120,11 @@ def setUp(self): github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None + self.orig_alternate_constants = copy.deepcopy(easyconfig.templates.ALTERNATE_TEMPLATE_CONSTANTS) + self.orig_alternate_templates = copy.deepcopy(easyconfig.templates.ALTERNATE_TEMPLATES) + self.orig_deprecated_constants = copy.deepcopy(easyconfig.templates.DEPRECATED_TEMPLATE_CONSTANTS) + self.orig_deprecated_templates = copy.deepcopy(easyconfig.templates.DEPRECATED_TEMPLATES) + def prep(self): """Prepare for test.""" # (re)cleanup last test file @@ -133,6 +138,13 @@ def prep(self): def tearDown(self): """ make sure to remove the temporary file """ st.get_cpu_architecture = self.orig_get_cpu_architecture + + easyconfig.templates.ALTERNATE_TEMPLATE_CONSTANTS = self.orig_alternate_template_constants + easyconfig.templates.ALTERNATE_TEMPLATES = self.orig_alternate_templates + easyconfig.templates.DEPRECATED_TEMPLATE_CONSTANTS = self.orig_deprecated_constants + easyconfig.templates.DEPRECATED_TEMPLATES = self.orig_deprecated_templates + reload(easyconfig.templates) + super(EasyConfigTest, self).tearDown() if os.path.exists(self.eb_file): os.remove(self.eb_file) @@ -1451,29 +1463,37 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) - def test_template_deprecation(self): - """Test deprecation of templates""" + def test_template_deprecation_and_alternate(self): + """Test deprecation of (and alternate) templates""" template_test_deprecations = { - 'builddir': ('new_build_dir', '1000000000'), - 'cudaver': ('new_cuda_ver', '1000000000'), - 'start_dir': ('new_start_dir', '1000000000'), + 'builddir': ('depr_build_dir', '1000000000'), + 'cudaver': ('depr_cuda_ver', '1000000000'), + 'start_dir': ('depr_start_dir', '1000000000'), } easyconfig.templates.DEPRECATED_TEMPLATES.update(template_test_deprecations) - tmpl_str = "cd %(start_dir)s && make PREFIX=%(installdir)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" + template_test_alternates = { + 'installdir': 'alt_install_dir', + 'version_maj_min': 'alt_ver_maj_min', + } + easyconfig.templates.ALTERNATE_TEMPLATES.update(template_test_deprecations) + + tmpl_str = ("cd %(start_dir)s && make PREFIX=%(installdir)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" + " && echo %(installdir)s %(version_maj_min)s" tmpl_dict = { - 'new_build_dir': '/example/build_dir', - 'new_cuda_ver': '12.1.1', + 'depr_build_dir': '/example/build_dir', + 'depr_cuda_ver': '12.1.1', 'installdir': '/example/installdir', - 'new_start_dir': '/example/build_dir/start_dir', + 'startdir': '/example/build_dir/start_dir', + 'alt_version_maj_min': '1.2', } with self.mocked_stdout_stderr() as (_, stderr): res = resolve_template(tmpl_str, tmpl_dict) stderr = stderr.getvalue() - for tmpl in ['builddir', 'cudaver', 'installdir', 'start_dir']: + for tmpl in [*template_test_deprecations.keys(), *template_test_alternates.keys()]: self.assertNotIn("%(" + tmpl + ")s", res) for old, (new, ver) in template_test_deprecations.items(): From 7e57dea99f2d56479688fe5f768aec98c37bd591 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Mon, 22 Apr 2024 05:59:55 +0100 Subject: [PATCH 245/430] fix typo --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index e34d8eb960..1bd0275097 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1480,7 +1480,7 @@ def test_template_deprecation_and_alternate(self): easyconfig.templates.ALTERNATE_TEMPLATES.update(template_test_deprecations) tmpl_str = ("cd %(start_dir)s && make PREFIX=%(installdir)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" - " && echo %(installdir)s %(version_maj_min)s" + " && echo %(installdir)s %(version_maj_min)s") tmpl_dict = { 'depr_build_dir': '/example/build_dir', 'depr_cuda_ver': '12.1.1', From 578f83b1e3a147f306186939307bbd3f80030b82 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Mon, 22 Apr 2024 06:07:10 +0100 Subject: [PATCH 246/430] fixes --- test/framework/easyconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 1bd0275097..26c0c79743 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -139,7 +139,7 @@ def tearDown(self): """ make sure to remove the temporary file """ st.get_cpu_architecture = self.orig_get_cpu_architecture - easyconfig.templates.ALTERNATE_TEMPLATE_CONSTANTS = self.orig_alternate_template_constants + easyconfig.templates.ALTERNATE_TEMPLATE_CONSTANTS = self.orig_alternate_constants easyconfig.templates.ALTERNATE_TEMPLATES = self.orig_alternate_templates easyconfig.templates.DEPRECATED_TEMPLATE_CONSTANTS = self.orig_deprecated_constants easyconfig.templates.DEPRECATED_TEMPLATES = self.orig_deprecated_templates @@ -1477,7 +1477,7 @@ def test_template_deprecation_and_alternate(self): 'installdir': 'alt_install_dir', 'version_maj_min': 'alt_ver_maj_min', } - easyconfig.templates.ALTERNATE_TEMPLATES.update(template_test_deprecations) + easyconfig.templates.ALTERNATE_TEMPLATES.update(template_test_alternates) tmpl_str = ("cd %(start_dir)s && make PREFIX=%(installdir)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" " && echo %(installdir)s %(version_maj_min)s") From 72291c81709771a0473793f3bf913113fc3989bb Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Mon, 22 Apr 2024 06:48:46 +0100 Subject: [PATCH 247/430] fix template test --- easybuild/framework/easyconfig/easyconfig.py | 20 ++++++++++++++------ test/framework/easyconfig.py | 15 ++++++++------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2857006a20..79fd4fec90 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1997,25 +1997,33 @@ def resolve_template(value, tmpl_dict): try: value = value % tmpl_dict except KeyError: - # check if any alternate or deprecated templates resolve + # check if any alternate and/or deprecated templates resolve try: orig_value = value # map old templates to new values for alternate and deprecated templates alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in ALTERNATE_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} + alt_map2 = {new_tmpl: tmpl_dict[old_tmpl] for (old_tmpl, new_tmpl) in + ALTERNATE_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()} depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in DEPRECATED_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} - # try templating with tmpl_dict, alt_map and depr_map - value = value % {**tmpl_dict, **alt_map, **depr_map} + + # try templating with alternate and deprecated templates included + value = value % {**tmpl_dict, **alt_map, **alt_map2, **depr_map} for old_tmpl, val in depr_map.items(): # check which deprecated templates were replaced, and issue deprecation warnings if old_tmpl in orig_value and val in value: new_tmpl, ver = DEPRECATED_TEMPLATES[old_tmpl] - _log.deprecated("Easyconfig template '%s' is deprecated, use '%s' instead" - % (old_tmpl, new_tmpl), ver) + _log.deprecated(f"Easyconfig template '{old_tmpl}' is deprecated, use '{new_tmpl}' instead", + ver) except KeyError: - _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict) + _log.warning(f"Unable to resolve template value {value} with dict {tmpl_dict}") + + for key in tmpl_dict: + if key in DEPRECATED_TEMPLATES: + new_key, ver = DEPRECATED_TEMPLATES[key] + _log.deprecated(f"Easyconfig template '{key}' is deprecated, use '{new_key}' instead", ver) else: # this block deals with references to objects and returns other references diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 26c0c79743..5587bc7b46 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1475,18 +1475,19 @@ def test_template_deprecation_and_alternate(self): template_test_alternates = { 'installdir': 'alt_install_dir', - 'version_maj_min': 'alt_ver_maj_min', + 'version_major_minor': 'alt_ver_maj_min', } easyconfig.templates.ALTERNATE_TEMPLATES.update(template_test_alternates) - tmpl_str = ("cd %(start_dir)s && make PREFIX=%(installdir)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" - " && echo %(installdir)s %(version_maj_min)s") + tmpl_str = ("cd %(start_dir)s && make %(namelower)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" + " && echo %(alt_install_dir)s %(version_major_minor)s") tmpl_dict = { 'depr_build_dir': '/example/build_dir', 'depr_cuda_ver': '12.1.1', 'installdir': '/example/installdir', - 'startdir': '/example/build_dir/start_dir', - 'alt_version_maj_min': '1.2', + 'start_dir': '/example/build_dir/start_dir', + 'alt_ver_maj_min': '1.2', + 'namelower': 'foo', } with self.mocked_stdout_stderr() as (_, stderr): @@ -1497,8 +1498,8 @@ def test_template_deprecation_and_alternate(self): self.assertNotIn("%(" + tmpl + ")s", res) for old, (new, ver) in template_test_deprecations.items(): - depr_str = (f"WARNING: Deprecated functionality, will no longer work in v{ver}: Easyconfig template '{old}'" - f" is deprecated, use '{new}' instead") + depr_str = (f"WARNING: Deprecated functionality, will no longer work in EasyBuild v{ver}: " + f"Easyconfig template '{old}' is deprecated, use '{new}' instead") self.assertIn(depr_str, stderr) def test_constant_doc(self): From 16367240bb95c65b24152539fefa5095404f03f9 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 22 Apr 2024 17:03:46 +0000 Subject: [PATCH 248/430] `get_software_libdir`: return dir if it's only 1 with libs There are situations e.g. with recent Gentoo where all 64-bit libraries are under `lib64`, but `lib` is seperate and used for other files and directories (e.g. `lib/python`), and for 32-bit libraries. `get_software_libdir` would fail for such. To avoid this: if `only_one` is `True` (default), *and* no specific file (`fs`) is specified (also default), *and* `lib` and `lib64` both exist and are separate, and only one of them has libraries (shared or static), it returns that one directory. --- easybuild/tools/modules.py | 10 ++++++++++ test/framework/modules.py | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index fc21a564b9..a24ddaa369 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -37,6 +37,7 @@ * Jens Timmerman (Ghent University) * David Brown (Pacific Northwest National Laboratory) """ +import glob import os import re import shlex @@ -52,6 +53,7 @@ from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.py2vs3 import subprocess_popen_text from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import get_subclasses, nub # software root/version environment variable name prefixes @@ -1671,6 +1673,7 @@ def get_software_libdir(name, only_one=True, fs=None): Returns the library subdirectory, relative to software root. It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs. + If only_one is True and fs is None, select the one subdirectory with shared or static libraries, if possible. :param name: name of the software package :param only_one: indicates whether only one lib path is expected to be found @@ -1703,6 +1706,13 @@ def get_software_libdir(name, only_one=True, fs=None): if len(res) == 1: res = res[0] else: + if fs is None: + # check if only one (exactly) has libraries + # this is needed for software with library archives in lib64 but other files/directories in lib + lib_glob = ["*.%s" % ext for ext in ["a", get_shared_lib_ext()]] + haslibs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res] + if haslibs[0] != haslibs[1]: + return res[int(not haslibs[0])] raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s", name, root, ', '.join(res)) return res diff --git a/test/framework/modules.py b/test/framework/modules.py index 82d03f6440..bb11802db7 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -674,10 +674,15 @@ def test_get_software_root_version_libdir(self): os.environ.pop('EBROOT%s' % env_var_name) os.environ.pop('EBVERSION%s' % env_var_name) - # check expected result of get_software_libdir with multiple lib subdirs + # if only 'lib' has a library archive, use it root = os.path.join(tmpdir, name) mkdir(os.path.join(root, 'lib64')) os.environ['EBROOT%s' % env_var_name] = root + write_file(os.path.join(root, 'lib', 'foo.a'), 'foo') + self.assertEqual(get_software_libdir(name), 'lib') + + # check expected result of get_software_libdir with multiple lib subdirs + remove_file(os.path.join(root, 'lib', 'foo.a')) self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name) self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64']) From 38c5541526f395e7119338fb1b6af65b5c31fde9 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 23 Apr 2024 09:59:27 +0100 Subject: [PATCH 249/430] fix alternate parameters --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/parser.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 79fd4fec90..7d0734e229 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -661,7 +661,7 @@ def set_keys(self, params): with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config - if key in self._config.keys() or key in DEPRECATED_PARAMETERS.keys(): + if any(key in x.keys() for x in (self._config,ALTERNATE_PARAMTERS, DEPRECATED_PARAMETERS)): self[key] = params[key] self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index 0a177c089a..5cab3b3ddf 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -44,7 +44,7 @@ # alternate easyconfig parameters, and their non-deprecated equivalents ALTERNATE_PARAMETERS = { - # : , + # : , } # deprecated easyconfig parameters, and their replacements From 26d3ec0e32d75489cc757dc0ea58b34ebead087d Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 23 Apr 2024 10:06:11 +0100 Subject: [PATCH 250/430] fix alternate parameters/templates --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/templates.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 7d0734e229..6a974cf0b5 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -661,7 +661,7 @@ def set_keys(self, params): with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config - if any(key in x.keys() for x in (self._config,ALTERNATE_PARAMTERS, DEPRECATED_PARAMETERS)): + if any(key in x.keys() for x in (self._config,ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS)): self[key] = params[key] self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 00d995a4b9..7ee17d2366 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -163,7 +163,7 @@ # alternate templates, and their equivalents ALTERNATE_TEMPLATES = { - # : , + # : , } # deprecated templates, and their replacements @@ -173,7 +173,7 @@ # alternate template constants, and their equivalents ALTERNATE_TEMPLATE_CONSTANTS = { - # : , + # : , } # deprecated template constants, and their replacements From 913f6c92c1607934ec4505397e08e1e6aa5e7fdd Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 23 Apr 2024 10:11:23 +0100 Subject: [PATCH 251/430] call self.prep at start of test --- test/framework/easyconfig.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 5587bc7b46..b29c675a78 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1466,6 +1466,8 @@ def test_sysroot_template(self): def test_template_deprecation_and_alternate(self): """Test deprecation of (and alternate) templates""" + self.prep() + template_test_deprecations = { 'builddir': ('depr_build_dir', '1000000000'), 'cudaver': ('depr_cuda_ver', '1000000000'), From e3f2cd49bfe70d3c66e7744aaf7a5edc83989c63 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 23 Apr 2024 10:15:51 +0100 Subject: [PATCH 252/430] whitespace --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 6a974cf0b5..1a98217556 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -661,7 +661,7 @@ def set_keys(self, params): with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config - if any(key in x.keys() for x in (self._config,ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS)): + if any(key in x.keys() for x in (self._config, ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS)): self[key] = params[key] self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) From b6fdaf5a17cc4bad3fec25396a68cc301a4cf5ca Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 24 Apr 2024 09:57:03 +0200 Subject: [PATCH 253/430] always include rpath configuration setting in output of --show-config --- easybuild/tools/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 54e9b81982..f5d5c801a0 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1387,7 +1387,8 @@ def show_config(self): # options that should never/always be printed ignore_opts = ['show_config', 'show_full_config'] - include_opts = ['buildpath', 'containerpath', 'installpath', 'repositorypath', 'robot_paths', 'sourcepath'] + include_opts = ['buildpath', 'containerpath', 'installpath', 'repositorypath', 'robot_paths', + 'rpath', 'sourcepath'] cmdline_opts_dict = self.dict_by_prefix() def reparse_cfg(args=None, withcfg=True): From 4fd2c83fb1679e0727902600ea01512a88afb00e Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Apr 2024 01:29:16 +0200 Subject: [PATCH 254/430] reset file timestamps in reproducible archives to unix epoch zero --- easybuild/tools/filetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 25813b43be..d9a88e9ff7 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2680,8 +2680,8 @@ def get_source_tarball_from_git(filename, target_dir, git_config): tar_cmd = [ # print names of all files and folders excluding .git directory 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', - # reset access and modification timestamps - '-exec', 'touch', '-t 197001010100', '{}', r'\;', '|', + # reset access and modification timestamps to epoch 0 + '-exec', 'touch', '--date=@0', '{}', r'\;', '|', # sort file list 'LC_ALL=C', 'sort', '--zero-terminated', '|', # create tarball in GNU format with ownership reset From c588c780b8530948c119840054a9d0b8694d4564 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Apr 2024 01:49:56 +0200 Subject: [PATCH 255/430] omit irrelevant file permissions in reproducible archives --- easybuild/tools/filetools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d9a88e9ff7..53ace623aa 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2684,9 +2684,9 @@ def get_source_tarball_from_git(filename, target_dir, git_config): '-exec', 'touch', '--date=@0', '{}', r'\;', '|', # sort file list 'LC_ALL=C', 'sort', '--zero-terminated', '|', - # create tarball in GNU format with ownership reset - 'tar', '--create', '--no-recursion', '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', - '--null', '--files-from', '-', '|', + # create tarball in GNU format with ownership and permissions reset + 'tar', '--create', '--no-recursion', '--owner=0', '--group=0', '--numeric-owner', '--mode="go+u,go-w"', + '--format=gnu', '--null', '--files-from', '-', '|', # compress tarball with gzip without original file name and timestamp 'gzip', '--no-name', '>', archive_path ] From ad674d3e32008a3f64ba59a39463ff35dbff61e4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 26 Apr 2024 17:21:15 +0200 Subject: [PATCH 256/430] fix test_show_config since RPATH setting is now always shown in output of --show-config --- test/framework/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index c401105116..84c6160702 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -65,7 +65,7 @@ from easybuild.tools.options import set_up_configuration, set_tmpdir, use_color from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX from easybuild.tools.run import run_shell_cmd -from easybuild.tools.systemtools import HAVE_ARCHSPEC +from easybuild.tools.systemtools import DARWIN, HAVE_ARCHSPEC, get_os_type from easybuild.tools.version import VERSION from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config @@ -5194,6 +5194,7 @@ def test_show_config(self): r"installpath\s* \(E\) = " + os.path.join(self.test_prefix, 'tmp.*'), r"repositorypath\s* \(D\) = " + os.path.join(default_prefix, 'ebfiles_repo'), r"robot-paths\s* \(E\) = " + os.path.join(test_dir, 'easyconfigs', 'test_ecs'), + r"rpath\s* \(D\) = " + ('False' if get_os_type() == DARWIN else 'True'), r"sourcepath\s* \(E\) = " + os.path.join(test_dir, 'sandbox', 'sources'), r"subdir-modules\s* \(F\) = mods", ] From ff82fd6602d5f9f8ef637f052a23a1fa649fc1f0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sat, 27 Apr 2024 13:44:29 +0200 Subject: [PATCH 257/430] Simplify easyblock.py - Remove `pass` statements where not required - Enhance docstrings - Use `.items` when values of dict is used (faster, clearer) - `basename` instead of `split()[-1]` --- easybuild/framework/easyblock.py | 41 +++++++++++++------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 33110c61dd..1f808c58e4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1365,7 +1365,7 @@ def make_module_dep(self, unload_info=None): multi_dep_mod_names[dep['name']].append(dep['short_mod_name']) multi_dep_load_defaults = [] - for depname, depmods in sorted(multi_dep_mod_names.items()): + for _, depmods in sorted(multi_dep_mod_names.items()): stmt = self.module_generator.load_module(depmods[0], multi_dep_mods=depmods, recursive_unload=recursive_unload, depends_on=depends_on) @@ -1769,10 +1769,7 @@ def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True): return ext_sep.join(exts_list) def prepare_for_extensions(self): - """ - Also do this before (eg to set the template) - """ - pass + """Ran before installing extensions (eg to set templates)""" def skip_extensions(self): """ @@ -2198,9 +2195,9 @@ def handle_iterate_opts(self): self.log.info("Current iteration index: %s", self.iter_idx) # pop first element from all iterative easyconfig parameters as next value to use - for opt in self.iter_opts: - if len(self.iter_opts[opt]) > self.iter_idx: - self.cfg[opt] = self.iter_opts[opt][self.iter_idx] + for opt, value in self.iter_opts.items(): + if len(value) > self.iter_idx: + self.cfg[opt] = value[self.iter_idx] else: self.cfg[opt] = '' # empty list => empty option as next value self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) @@ -2212,12 +2209,12 @@ def post_iter_step(self): """Restore options that were iterated over""" # disable templating, since we're messing about with values in self.cfg with self.cfg.disable_templating(): - for opt in self.iter_opts: - self.cfg[opt] = self.iter_opts[opt] + for opt, value in self.iter_opts.items(): + self.cfg[opt] = value # also need to take into account extensions, since those were iterated over as well for ext in self.ext_instances: - ext.cfg[opt] = self.iter_opts[opt] + ext.cfg[opt] = value self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) @@ -2751,10 +2748,7 @@ def _test_step(self): self.report_test_failure(err) def stage_install_step(self): - """ - Install in a stage directory before actual installation. - """ - pass + """Install in a stage directory before actual installation.""" def install_step(self): """Install built software (abstract method).""" @@ -3248,7 +3242,7 @@ def sanity_check_linked_shared_libs(self, subdirs=None): required_libs.extend(self.cfg['required_linked_shared_libs']) # early return if there are no banned/required libraries - if not (banned_libs + required_libs): + if not banned_libs + required_libs: self.log.info("No banned/required libraries specified") return [] else: @@ -4463,7 +4457,7 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): else: easyblock_paths.add(easyblock_path) for easyblock_path in easyblock_paths: - easyblock_basedir, easyblock_filename = os.path.split(easyblock_path) + easyblock_filename = os.path.basename(easyblock_path) copy_file(easyblock_path, os.path.join(reprod_easyblock_dir, easyblock_filename)) _log.info("Dumped easyblock %s required for reproduction to %s", easyblock_filename, reprod_easyblock_dir) @@ -4594,10 +4588,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results): class StopException(Exception): - """ - StopException class definition. - """ - pass + """Exception thrown to stop running steps""" def inject_checksums_to_json(ecs, checksum_type): @@ -4645,14 +4636,14 @@ def inject_checksums_to_json(ecs, checksum_type): # actually inject new checksums or overwrite existing ones (if --force) existing_checksums = app.get_checksums_from_json(always_read=True) - for filename in checksums: + for filename, checksum in checksums.items(): if filename not in existing_checksums: - existing_checksums[filename] = checksums[filename] + existing_checksums[filename] = checksum # don't do anything if the checksum already exist and is the same - elif checksums[filename] != existing_checksums[filename]: + elif checksum != existing_checksums[filename]: if build_option('force'): print_warning("Found existing checksums for %s, overwriting them (due to --force)..." % ec_fn) - existing_checksums[filename] = checksums[filename] + existing_checksums[filename] = checksum else: raise EasyBuildError("Found existing checksum for %s, use --force to overwrite them" % filename) From 27a6e93bc23b2f5f4379cb57ceca4ce669824816 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sat, 27 Apr 2024 16:20:42 +0200 Subject: [PATCH 258/430] check easyblocks and framework major are the same --- easybuild/main.py | 6 ++++++ easybuild/tools/version.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index 42bd83dff5..0f89335a98 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -83,6 +83,7 @@ from easybuild.tools.repository.repository import init_repository from easybuild.tools.systemtools import check_easybuild_deps from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state +from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION, different_major_versions _log = None @@ -618,6 +619,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, from_pr_list, tweaked_ecs_paths) = cfg_settings + # compare running Framework and EasyBlocks versions + if different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION): + raise EasyBuildError(f"Framework ({FRAMEWORK_VERSION}) and EasyBlock ({EASYBLOCKS_VERSION}) major versions " + "are different.") + # load hook implementations (if any) hooks = load_hooks(options.hooks) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index c01c8fc5c3..d9b2ce42a7 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -103,3 +103,16 @@ def this_is_easybuild(): msg = msg.encode('ascii') return msg + + +def different_major_versions(v1, v2): + """Compare major versions""" + # Deal with version instances being either strings or LooseVersion + if not isinstance(v1, LooseVersion): + v1 = LooseVersion(v1) + if not isinstance(v2, LooseVersion): + v2 = LooseVersion(v2) + + if v1.version[0] == v2.version[0]: + return False + return True From a91b1a4b0ed94a853e00a870a32b0dceaf5cc9e6 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sat, 27 Apr 2024 16:29:28 +0200 Subject: [PATCH 259/430] must support python < 3.6 --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 0f89335a98..c93f1dc212 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -621,8 +621,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr # compare running Framework and EasyBlocks versions if different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION): - raise EasyBuildError(f"Framework ({FRAMEWORK_VERSION}) and EasyBlock ({EASYBLOCKS_VERSION}) major versions " - "are different.") + raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION, + EASYBLOCKS_VERSION)) # load hook implementations (if any) hooks = load_hooks(options.hooks) From 31514bb9a4fa26ad3148340411cfbaf05cf1717f Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sat, 27 Apr 2024 16:36:42 +0200 Subject: [PATCH 260/430] deal with unknown blocks version --- easybuild/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index c93f1dc212..94bf7b46a8 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -620,7 +620,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr from_pr_list, tweaked_ecs_paths) = cfg_settings # compare running Framework and EasyBlocks versions - if different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION): + if 'UNKNOWN' in EASYBLOCKS_VERSION: + print_msg('Unable to determine EasyBlocks version, so we'll assume it is not different from Framework') + elif different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION): raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION, EASYBLOCKS_VERSION)) From 5202f07081133fbe55510664b885452eb8f7675e Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sat, 27 Apr 2024 16:37:16 +0200 Subject: [PATCH 261/430] " and ' mixing --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 94bf7b46a8..1e44609d4e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -621,7 +621,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr # compare running Framework and EasyBlocks versions if 'UNKNOWN' in EASYBLOCKS_VERSION: - print_msg('Unable to determine EasyBlocks version, so we'll assume it is not different from Framework') + print_msg("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework") elif different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION): raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION, EASYBLOCKS_VERSION)) From 2724c8e28c0f05179639bd69f56d65420633981c Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sat, 27 Apr 2024 16:45:38 +0200 Subject: [PATCH 262/430] fix --- easybuild/main.py | 4 ++-- easybuild/tools/version.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 1e44609d4e..6dda9dc893 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -83,7 +83,7 @@ from easybuild.tools.repository.repository import init_repository from easybuild.tools.systemtools import check_easybuild_deps from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state -from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION, different_major_versions +from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION, UNKNOWN_VERSION, different_major_versions _log = None @@ -620,7 +620,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr from_pr_list, tweaked_ecs_paths) = cfg_settings # compare running Framework and EasyBlocks versions - if 'UNKNOWN' in EASYBLOCKS_VERSION: + if EASYBLOCKS_VERSION == UNKNOWN_VERSION: print_msg("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework") elif different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION): raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION, diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index d9b2ce42a7..63275b69ff 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -47,6 +47,7 @@ # This causes problems further up the dependency chain... VERSION = LooseVersion('4.9.2.dev0') UNKNOWN = 'UNKNOWN' +UNKNOWN_VERSION = '0.0.UNKNOWN.EASYBLOCKS' def get_git_revision(): @@ -87,7 +88,7 @@ def get_git_revision(): try: from easybuild.easyblocks import VERBOSE_VERSION as EASYBLOCKS_VERSION except Exception: - EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' # make sure it is smaller then anything + EASYBLOCKS_VERSION = UNKNOWN_VERSION # make sure it is smaller then anything def this_is_easybuild(): From 1a2c099884ebc334f727f040b5a501a18155406f Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 29 Apr 2024 16:20:52 +0200 Subject: [PATCH 263/430] use generic chmod command to reset permissions of reproducible archives --- easybuild/tools/filetools.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 53ace623aa..c630c75bca 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2680,12 +2680,14 @@ def get_source_tarball_from_git(filename, target_dir, git_config): tar_cmd = [ # print names of all files and folders excluding .git directory 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', - # reset access and modification timestamps to epoch 0 - '-exec', 'touch', '--date=@0', '{}', r'\;', '|', - # sort file list + # reset access and modification timestamps to epoch 0 (equivalent to --mtime in GNU tar) + '-exec', 'touch', '--date=@0', '{}', r'\;', + # reset file permissions of cloned repo (equivalent to --mode in GNU tar) + '-exec', 'chmod', '"go+u,go-w"', '{}', r'\;', '|', + # sort file list (equivalent to --sort in GNU tar) 'LC_ALL=C', 'sort', '--zero-terminated', '|', # create tarball in GNU format with ownership and permissions reset - 'tar', '--create', '--no-recursion', '--owner=0', '--group=0', '--numeric-owner', '--mode="go+u,go-w"', + 'tar', '--create', '--no-recursion', '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', '--null', '--files-from', '-', '|', # compress tarball with gzip without original file name and timestamp 'gzip', '--no-name', '>', archive_path From 0a5c99f04d3d448974825e1d88f443c496a181e8 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Tue, 30 Apr 2024 19:39:43 +0100 Subject: [PATCH 264/430] address review --- easybuild/main.py | 3 ++- easybuild/tools/version.py | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 6dda9dc893..4ea569bbd4 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -60,6 +60,7 @@ from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak +from easybuild.tools.build_log import print_warning from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software @@ -621,7 +622,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr # compare running Framework and EasyBlocks versions if EASYBLOCKS_VERSION == UNKNOWN_VERSION: - print_msg("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework") + print_warning("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework") elif different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION): raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION, EASYBLOCKS_VERSION)) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 63275b69ff..3b0c45d2bf 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -109,11 +109,9 @@ def this_is_easybuild(): def different_major_versions(v1, v2): """Compare major versions""" # Deal with version instances being either strings or LooseVersion - if not isinstance(v1, LooseVersion): + if isinstance(v1, str): v1 = LooseVersion(v1) - if not isinstance(v2, LooseVersion): + if isinstance(v2, str): v2 = LooseVersion(v2) - if v1.version[0] == v2.version[0]: - return False - return True + return v1.version[0] != v2.version[0] From d8be5ca9cde44780bae46f76a791026ef2c8161b Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Tue, 30 Apr 2024 19:40:48 +0100 Subject: [PATCH 265/430] remove unneeded import --- easybuild/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 4ea569bbd4..00b2899227 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -60,7 +60,6 @@ from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak -from easybuild.tools.build_log import print_warning from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software From 5689ad37c6808fdeadb052e027ddd203b2b85608 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 3 May 2024 09:25:43 +0100 Subject: [PATCH 266/430] switch checksum default to sha256 --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index c630c75bca..ded4d4e361 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -121,7 +121,7 @@ CHECKSUM_TYPE_MD5 = 'md5' CHECKSUM_TYPE_SHA256 = 'sha256' -DEFAULT_CHECKSUM = CHECKSUM_TYPE_MD5 +DEFAULT_CHECKSUM = CHECKSUM_TYPE_SHA256 # map of checksum types to checksum functions CHECKSUM_FUNCTIONS = { From 45a05454f64ec1963a63ce96a2c93110a12e6419 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Fri, 3 May 2024 09:38:52 +0100 Subject: [PATCH 267/430] paths --- easybuild/tools/github.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ae9b78b5ed..9a70109c00 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -587,6 +587,11 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi else: raise EasyBuildError("Couldn't find path to patched file %s", full_path) + if github_repo == GITHUB_EASYCONFIGS_REPO: + print('get ecs version', os.path.join(final_path, 'setup.py')) + elif github_repo == GITHUB_EASYBLOCKS_REPO: + print('get blocks version', sys.path, final_path) + return files From 3d4aa2623fcba1ae8c4060720a3330a0f7fcc9a9 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Fri, 3 May 2024 09:44:17 +0100 Subject: [PATCH 268/430] update test for new default --- test/framework/filetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 064d8c09df..ab1b8fbf7b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -303,8 +303,8 @@ def test_checksums(self): self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum) self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum))) - # default checksum type is MD5 - self.assertEqual(ft.compute_checksum(fp), known_checksums['md5']) + # default checksum type is SHA256 + self.assertEqual(ft.compute_checksum(fp), known_checksums['sha256']) # both MD5 and SHA256 checksums can be verified without specifying type self.assertTrue(ft.verify_checksum(fp, known_checksums['md5'])) From 3757e9d51713a73ab4879068b46be3800947ca30 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Fri, 3 May 2024 10:29:58 +0100 Subject: [PATCH 269/430] remove Python 2 compat in LooseVersion --- easybuild/tools/loose_version.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py index fb2e0fcb78..ea4fa69f6c 100644 --- a/easybuild/tools/loose_version.py +++ b/easybuild/tools/loose_version.py @@ -10,12 +10,7 @@ # - Changes to documentation and formatting import re -# Modified: Make this compatible with Python 2 -try: - from itertools import zip_longest -except ImportError: - # Python 2 - from itertools import izip_longest as zip_longest +from itertools import zip_longest class LooseVersion(object): From 70534b43c9f4b343939a01090bd6b736d6e8405d Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 15:28:29 +0200 Subject: [PATCH 270/430] add new method tools.filetools.get_cwd to retrieve current working directory --- easybuild/tools/filetools.py | 14 ++++++++++++++ test/framework/filetools.py | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ded4d4e361..769fb0c988 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -406,6 +406,20 @@ def remove(paths): else: raise EasyBuildError("Specified path to remove is not an existing file or directory: %s", path) +def get_cwd(must_exist=True): + """ + Retrieve current working directory + """ + try: + cwd = os.getcwd() + except FileNotFoundError as err: + if must_exist is True: + raise EasyBuildError("Working directory does not exist") + else: + _log.debug("Failed to determine current working directory, but proceeding anyway: %s", err) + cwd = None + + return cwd def change_dir(path): """ diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ab1b8fbf7b..bff8b70ba9 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2222,6 +2222,19 @@ def test_copy(self): self.assertTrue(os.path.isfile(os.path.join(self.test_prefix, 'GCC-4.6.3.eb'))) self.assertEqual(txt, '') + def test_get_cwd(self): + """Test get_cwd""" + toy_dir = os.path.join(self.test_prefix, "test_get_cwd_dir") + os.mkdir(toy_dir) + os.chdir(toy_dir) + + self.assertTrue(os.path.samefile(ft.get_cwd(), toy_dir)) + + os.rmdir(toy_dir) + self.assertErrorRegex(EasyBuildError, "Working directory does not exist", ft.get_cwd) + + self.assertEqual(ft.get_cwd(must_exist=False), None) + def test_change_dir(self): """Test change_dir""" From acf02e30dcaa3ccf74674f1be64bf651c4bfc967 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 15:29:07 +0200 Subject: [PATCH 271/430] use tools.filetools.get_cwd in tools.filetools.change_dir --- easybuild/tools/filetools.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 769fb0c988..86db5817f8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -428,19 +428,18 @@ def change_dir(path): :param path: location to change to :return: previous location we were in """ - # determining the current working directory can fail if we're in a non-existing directory - try: - cwd = os.getcwd() - except OSError as err: - _log.debug("Failed to determine current working directory (but proceeding anyway: %s", err) - cwd = None + # determine origin working directory: can fail if non-existent + prev_dir = get_cwd(must_exist=False) try: os.chdir(path) except OSError as err: - raise EasyBuildError("Failed to change from %s to %s: %s", cwd, path, err) + raise EasyBuildError("Failed to change from %s to %s: %s", prev_dir, path, err) - return cwd + # determine final working directory: must exist + get_cwd() + + return prev_dir def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced=False, change_into_dir=False, From 912fac0c25e24e8297e2b889ad7d0f81c2d19954 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 15:36:03 +0200 Subject: [PATCH 272/430] use filetools.get_cwd in filetools.find_base_dir and filetools.parse_http_header_fields_urlpat --- easybuild/tools/filetools.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 86db5817f8..142d048dad 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -684,9 +684,9 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header if argline == '' or '#' in argline[0]: continue # permit comment lines: ignore them - if os.path.isfile(os.path.join(os.getcwd(), argline)): + if os.path.isfile(os.path.join(get_cwd(), argline)): # expand existing relative path to absolute - argline = os.path.join(os.path.join(os.getcwd(), argline)) + argline = os.path.join(os.path.join(get_cwd(), argline)) if os.path.isfile(argline): # argline is a file path, so read that instead _log.debug('File included in parse_http_header_fields_urlpat: %s' % argline) @@ -1341,14 +1341,14 @@ def get_local_dirs_purged(): # and hidden directories ignoredirs = ["easybuild"] - lst = os.listdir(os.getcwd()) + lst = os.listdir(get_cwd()) lst = [d for d in lst if not d.startswith('.') and d not in ignoredirs] return lst lst = get_local_dirs_purged() - new_dir = os.getcwd() + new_dir = get_cwd() while len(lst) == 1: - new_dir = os.path.join(os.getcwd(), lst[0]) + new_dir = os.path.join(get_cwd(), lst[0]) if not os.path.isdir(new_dir): break From 9ed35ce59089619c3d61a2e3f673c638c88a269a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 15:59:20 +0200 Subject: [PATCH 273/430] use filetools.get_cwd in EasyBlock class --- easybuild/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index bf9c939231..d95c1622b7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -81,8 +81,8 @@ from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file -from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url +from easybuild.tools.filetools import encode_class_name, extract_file, find_backup_name_candidate +from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP @@ -151,7 +151,7 @@ def __init__(self, ec): """ # keep track of original working directory, so we can go back there - self.orig_workdir = os.getcwd() + self.orig_workdir = get_cwd() # dict of all hooks (mapping of name to function) self.hooks = load_hooks(build_option('hooks')) @@ -3725,7 +3725,7 @@ def cleanup_step(self): # make sure we're out of the dir we're removing change_dir(self.orig_workdir) - self.log.info("Cleaning up builddir %s (in %s)", self.builddir, os.getcwd()) + self.log.info("Cleaning up builddir %s (in %s)", self.builddir, get_cwd()) try: remove_dir(self.builddir) @@ -4249,7 +4249,7 @@ def build_and_install_one(ecdict, init_env): restore_env(init_env) sanitize_env() - cwd = os.getcwd() + cwd = get_cwd() # load easyblock easyblock = build_option('easyblock') @@ -4554,7 +4554,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results): instance = get_easyblock_instance(ec) apps.append(instance) - base_dir = os.getcwd() + base_dir = get_cwd() # keep track of environment right before initiating builds # note: may be different from ORIG_OS_ENVIRON, since EasyBuild may have defined additional env vars itself by now From 9ca0937f89f4d9c47f7594eea6060d7dbe36afc2 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:02:40 +0200 Subject: [PATCH 274/430] use filetools.get_cwd in easyconfig.tools --- easybuild/framework/easyconfig/tools.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index c354bec3a9..2da3dc9d96 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -57,7 +57,7 @@ from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files +from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files from easybuild.tools.filetools import read_file, resolve_path, which, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit @@ -792,14 +792,13 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None): target_path, paths = None, [] - # if only one argument is specified, use current directory as target directory if len(orig_paths) == 1: - target_path = os.getcwd() + # if only one argument is specified, use current directory as target directory + target_path = get_cwd() paths = orig_paths[:] - - # if multiple arguments are specified, assume that last argument is target location, - # and remove that from list of paths to copy elif orig_paths: + # if multiple arguments are specified, assume that last argument is target location, + # and remove that from list of paths to copy target_path = orig_paths[-1] paths = orig_paths[:-1] @@ -817,7 +816,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None): pr_paths.extend(fetch_files_from_pr(pr=pr, path=tmpdir)) # assume that files need to be copied to current working directory for now - target_path = os.getcwd() + target_path = get_cwd() if orig_paths: last_path = orig_paths[-1] @@ -854,7 +853,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None): commit_paths = fetch_files_from_commit(from_commit, path=tmpdir) # assume that files need to be copied to current working directory for now - target_path = os.getcwd() + target_path = get_cwd() if orig_paths: last_path = orig_paths[-1] From 85ef1dec85f02a383cd1a6320d5cccf21ffbaf92 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:08:06 +0200 Subject: [PATCH 275/430] use filetools.get_cwd in easyconfig.tools.run --- easybuild/tools/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index ca33a94a9c..3fa471dcb2 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -65,6 +65,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import build_option +from easybuild.tools.filetools import get_cwd from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.utilities import trace_msg @@ -315,7 +316,7 @@ def to_cmd_str(cmd): qa_wait_patterns = [] if work_dir is None: - work_dir = os.getcwd() + work_dir = get_cwd() cmd_str = to_cmd_str(cmd) From a5e2db19c7769861c01f913dc7d7ea413fbdcd71 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:08:30 +0200 Subject: [PATCH 276/430] use filetools.get_cwd in easybuild.tools.parallelbuild --- easybuild/tools/parallelbuild.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 3ab31e1efc..b25d8b9bdc 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -43,6 +43,7 @@ from easybuild.framework.easyconfig.easyconfig import ActiveMNS from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_repository, get_repositorypath +from easybuild.tools.filetools import get_cwd from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.job.backend import job_backend from easybuild.tools.repository.repository import init_repository @@ -126,7 +127,7 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True): :param testing: If `True`, skip actual job submission :param prepare_first: prepare by runnning fetch step first for each easyconfig """ - curdir = os.getcwd() + curdir = get_cwd() # regex pattern for options to ignore (help options can't reach here) ignore_opts = re.compile('^--robot$|^--job|^--try-.*$|^--easystack$') From b3c424dfda039d88fb17b4922457b1b1facdb348 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:10:04 +0200 Subject: [PATCH 277/430] use filetools.get_cwd in easybuild.tools.options --- easybuild/tools/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 54e9b81982..8b86cc8fb2 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -84,8 +84,8 @@ from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.environment import restore_env, unset_env_vars -from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, install_fake_vsc -from easybuild.tools.filetools import move_file, which +from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, get_cwd +from easybuild.tools.filetools import install_fake_vsc, move_file, which from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS @@ -829,7 +829,7 @@ def job_options(self): 'eb-cmd': ("EasyBuild command to use in jobs", 'str', 'store', DEFAULT_JOB_EB_CMD), 'max-jobs': ("Maximum number of concurrent jobs (queued and running, 0 = unlimited)", 'int', 'store', 0), 'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24), - 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()), + 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', get_cwd()), 'polling-interval': ("Interval between polls for status of jobs (in seconds)", float, 'store', 30.0), 'target-resource': ("Target resource for jobs", None, 'store', None), }) From a97d2aea80cee684338a96a334768958b59232ae Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:10:59 +0200 Subject: [PATCH 278/430] use filetools.get_cwd in easybuild.tools.testing --- easybuild/tools/testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 881788337e..359150ef6c 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -47,7 +47,7 @@ from easybuild.framework.easyconfig.tools import skip_available from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file, write_file +from easybuild.tools.filetools import find_easyconfigs, get_cwd, mkdir, read_file, write_file from easybuild.tools.github import GITHUB_EASYBLOCKS_REPO, GITHUB_EASYCONFIGS_REPO, create_gist, post_comment_in_issue from easybuild.tools.jenkins import aggregate_xml_in_dirs from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel @@ -67,7 +67,7 @@ def regtest(easyconfig_paths, modtool, build_specs=None): :param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ - cur_dir = os.getcwd() + cur_dir = get_cwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: From 17ea12a83f8264805ade061b836810029d6cf23f Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:12:02 +0200 Subject: [PATCH 279/430] use filetools.get_cwd in easybuild.tools.robot --- easybuild/tools/robot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 5189769929..a5421b7225 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -45,7 +45,7 @@ from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import det_common_path_prefix, search_file +from easybuild.tools.filetools import get_cwd, det_common_path_prefix, search_file from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.utilities import flatten, nub @@ -491,7 +491,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con """ search_path = build_option('robot_path') if not search_path: - search_path = [os.getcwd()] + search_path = [get_cwd()] extra_search_paths = build_option('search_paths') # If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths if extra_search_paths and consider_extra_paths: From 62fc71d3fd96327f293a3d631ad2b2497d425ca2 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:13:18 +0200 Subject: [PATCH 280/430] use filetools.get_cwd in easybuild.base.optcomplete --- easybuild/base/optcomplete.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index 7a46c49921..ba0f075cf2 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -107,6 +107,7 @@ from optparse import OptionParser, Option from pprint import pformat +from easybuild.tools.filetools import get_cwd from easybuild.tools.utilities import shell_quote debugfn = None # for debugging only @@ -537,7 +538,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete # Note: this will get filtered properly below. completer_kwargs = { - 'pwd': os.getcwd(), + 'pwd': get_cwd(), 'cline': cline, 'cpoint': cpoint, 'prefix': prefix, From 3ebc2f3490ed97bc15d27f5269f86f03f1e6823b Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:15:17 +0200 Subject: [PATCH 281/430] use filetools.get_cwd in easybuild.tools.jobs.pbs_python --- easybuild/tools/job/pbs_python.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index bf90b57063..31199dd1ee 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -39,6 +39,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option +from easybuild.tools.filetools import get_cwd from easybuild.tools.job.backend import JobBackend from easybuild.tools.utilities import only_if_module_is_available @@ -320,8 +321,8 @@ def _submit(self): self.log.debug("Job hold attributes: %s" % hold_attributes[0].value) # add a bunch of variables (added by qsub) - # also set PBS_O_WORKDIR to os.getcwd() - os.environ.setdefault('WORKDIR', os.getcwd()) + # also set PBS_O_WORKDIR to current working dir + os.environ.setdefault('WORKDIR', get_cwd()) defvars = ['MAIL', 'HOME', 'PATH', 'SHELL', 'WORKDIR'] pbsvars = ["PBS_O_%s=%s" % (x, os.environ.get(x, 'NOTFOUND_%s' % x)) for x in defvars] From 9273213b96e67709906887c704eb98b8df70b133 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:15:58 +0200 Subject: [PATCH 282/430] use filetools.get_cwd in easybuild.tools.repository.svnrepo --- easybuild/tools/repository/svnrepo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index 79f593b1ee..1e5d980f3b 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -46,7 +46,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import remove_dir +from easybuild.tools.filetools import get_cwd, remove_dir from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.utilities import only_if_module_is_available @@ -145,7 +145,7 @@ def stage_file(self, path): """ if self.client and not self.client.status(path)[0].is_versioned: # add it to version control - self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, os.getcwd())) + self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, get_cwd())) self.client.add(path) def add_easyconfig(self, cfg, name, version, stats, previous_stats): From a2fee23aa81740dd44379cd24d80d0fcbc3eaec8 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:20:05 +0200 Subject: [PATCH 283/430] fix formatting around tools.filetools.get_cwd --- easybuild/tools/filetools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 142d048dad..d8990560b1 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -406,6 +406,7 @@ def remove(paths): else: raise EasyBuildError("Specified path to remove is not an existing file or directory: %s", path) + def get_cwd(must_exist=True): """ Retrieve current working directory @@ -421,6 +422,7 @@ def get_cwd(must_exist=True): return cwd + def change_dir(path): """ Change to directory at specified location. From cc902931c0b59172446c952a496cd546421fe5d0 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:28:51 +0200 Subject: [PATCH 284/430] easyconfig.tools.run is too low to use higher level method from filetools This reverts commit 85ef1dec85f02a383cd1a6320d5cccf21ffbaf92. --- easybuild/tools/run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 3fa471dcb2..ca33a94a9c 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -65,7 +65,6 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import build_option -from easybuild.tools.filetools import get_cwd from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.utilities import trace_msg @@ -316,7 +315,7 @@ def to_cmd_str(cmd): qa_wait_patterns = [] if work_dir is None: - work_dir = get_cwd() + work_dir = os.getcwd() cmd_str = to_cmd_str(cmd) From e6890ca7ee3be8a3ee58176f6beee14ac1533491 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:38:43 +0200 Subject: [PATCH 285/430] improve error on FileNotFoundError on os.getcwd in run_shell_cmd --- easybuild/tools/run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index ca33a94a9c..6a3d29972d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -315,7 +315,10 @@ def to_cmd_str(cmd): qa_wait_patterns = [] if work_dir is None: - work_dir = os.getcwd() + try: + work_dir = os.getcwd() + except FileNotFoundError: + raise EasyBuildError("Working directory does not exist") cmd_str = to_cmd_str(cmd) From f17e65d4dcdb6ad4a0b00cb4ab5782bc036c9d73 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 5 May 2024 13:34:23 +0100 Subject: [PATCH 286/430] deprecate old checksum options (incl. md5) --- easybuild/framework/easyblock.py | 12 +-- easybuild/framework/easyconfig/types.py | 2 +- easybuild/tools/filetools.py | 10 +- .../test_ecs/t/toy/toy-0.0-deps.eb | 9 +- .../test_ecs/t/toy/toy-0.0-test.eb | 9 +- .../easyconfigs/test_ecs/t/toy/toy-0.0.eb | 9 +- test/framework/filetools.py | 97 ++++++++++++++----- test/framework/options.py | 24 ++--- 8 files changed, 108 insertions(+), 64 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index bf9c939231..51bccdc301 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -77,7 +77,7 @@ from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env -from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 +from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file @@ -666,8 +666,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): src_path = ext_src['src'] src_fn = os.path.basename(src_path) - # report both MD5 and SHA256 checksums, since both are valid default checksum types - for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256): + for checksum_type in [CHECKSUM_TYPE_SHA256]: src_checksum = compute_checksum(src_path, checksum_type=checksum_type) self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum) @@ -695,9 +694,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if verify_checksums: for patch in ext_patches: patch = patch['path'] - # report both MD5 and SHA256 checksums, - # since both are valid default checksum types - for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256): + for checksum_type in [CHECKSUM_TYPE_SHA256]: checksum = compute_checksum(patch, checksum_type=checksum_type) self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum) @@ -2410,8 +2407,7 @@ def fetch_step(self, skip_checksums=False): # compute checksums for all source and patch files if not (skip_checksums or self.dry_run): for fil in self.src + self.patches: - # report both MD5 and SHA256 checksums, since both are valid default checksum types - for checksum_type in [CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256]: + for checksum_type in [CHECKSUM_TYPE_SHA256]: fil[checksum_type] = compute_checksum(fil['path'], checksum_type=checksum_type) self.log.info("%s checksum for %s: %s", checksum_type, fil['path'], fil[checksum_type]) diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index b7af38867e..f902716dfa 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -516,7 +516,7 @@ def to_checksums(checksums): for checksum in checksums: # each list entry can be: # * None (indicates no checksum) - # * a string (MD5 or SHA256 checksum) + # * a string (SHA256 checksum) # * a tuple with 2 elements: checksum type + checksum value # * a list of checksums (i.e. multiple checksums for a single file) # * a dict (filename to checksum mapping) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ded4d4e361..60fab80ef5 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1192,12 +1192,16 @@ def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM): Compute checksum of specified file. :param path: Path of file to compute checksum for - :param checksum_type: type(s) of checksum ('adler32', 'crc32', 'md5' (default), 'sha1', 'sha256', 'sha512', 'size') + :param checksum_type: type(s) of checksum ('adler32', 'crc32', 'md5', 'sha1', 'sha256', 'sha512', 'size') """ if checksum_type not in CHECKSUM_FUNCTIONS: raise EasyBuildError("Unknown checksum type (%s), supported types are: %s", checksum_type, CHECKSUM_FUNCTIONS.keys()) + if checksum_type in ['adler32', 'crc32', 'md5', 'sha1', 'size']: + _log.deprecated("Checksum type %s is deprecated. Use sha256 (default) or sha512 instead" % checksum_type, + '6.0') + try: checksum = CHECKSUM_FUNCTIONS[checksum_type](path) except IOError as err: @@ -1235,7 +1239,7 @@ def verify_checksum(path, checksums): Verify checksum of specified file. :param path: path of file to verify checksum of - :param checksums: checksum values (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b') + :param checksums: checksum values (and type, optionally, default is sha256), e.g., 'af314', ('sha', '5ec1b') """ filename = os.path.basename(path) @@ -1287,7 +1291,7 @@ def verify_checksum(path, checksums): # no matching checksums return False else: - raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), " + raise EasyBuildError("Invalid checksum spec '%s': should be a string (SHA256), " "2-tuple (type, value), or tuple of alternative checksum specs.", checksum) diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb index 4ae349d0a0..1e3f34a777 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb @@ -9,13 +9,10 @@ toolchain = SYSTEM sources = [SOURCE_TAR_GZ] checksums = [[ - 'be662daa971a640e40be5c804d9d7d10', # default (MD5) '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) - ('adler32', '0x998410035'), - ('crc32', '0x1553842328'), - ('md5', 'be662daa971a640e40be5c804d9d7d10'), - ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), - ('size', 273), + ('sha512', + '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0' + 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'), ]] patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb index 90cc7429d3..286cdf7f6c 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb @@ -9,13 +9,10 @@ toolchain = SYSTEM sources = [SOURCE_TAR_GZ] checksums = [[ - 'be662daa971a640e40be5c804d9d7d10', # default (MD5) '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) - ('adler32', '0x998410035'), - ('crc32', '0x1553842328'), - ('md5', 'be662daa971a640e40be5c804d9d7d10'), - ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), - ('size', 273), + ('sha512', + '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0' + 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'), ]] patches = [ 'toy-0.0_fix-silly-typo-in-printf-statement.patch', diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb index c2a88616b1..0e087cec7c 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb @@ -8,13 +8,10 @@ toolchain = SYSTEM sources = [SOURCE_TAR_GZ] checksums = [[ - 'be662daa971a640e40be5c804d9d7d10', # default (MD5) '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) - ('adler32', '0x998410035'), - ('crc32', '0x1553842328'), - ('md5', 'be662daa971a640e40be5c804d9d7d10'), - ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), - ('size', 273), + ('sha512', + '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0' + 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'), ]] patches = [ 'toy-0.0_fix-silly-typo-in-printf-statement.patch', diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ab1b8fbf7b..636503b9cc 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -289,10 +289,6 @@ def test_checksums(self): ft.write_file(fp, "easybuild\n") known_checksums = { - 'adler32': '0x379257805', - 'crc32': '0x1457143216', - 'md5': '7167b64b1ca062b9674ffef46f9325db', - 'sha1': 'db05b79e09a4cc67e9dd30b313b5488813db3190', 'sha256': '1c49562c4b404f3120a3fa0926c8d09c99ef80e470f7de03ffdfa14047960ea5', 'sha512': '7610f6ce5e91e56e350d25c917490e4815f7986469fafa41056698aec256733e' 'b7297da8b547d5e74b851d7c4e475900cec4744df0f887ae5c05bf1757c224b4', @@ -306,12 +302,10 @@ def test_checksums(self): # default checksum type is SHA256 self.assertEqual(ft.compute_checksum(fp), known_checksums['sha256']) - # both MD5 and SHA256 checksums can be verified without specifying type - self.assertTrue(ft.verify_checksum(fp, known_checksums['md5'])) + # SHA256 checksums can be verified without specifying type self.assertTrue(ft.verify_checksum(fp, known_checksums['sha256'])) - # providing non-matching MD5 and SHA256 checksums results in failed verification - self.assertFalse(ft.verify_checksum(fp, '1c49562c4b404f3120a3fa0926c8d09c')) + # providing non-matching SHA256 checksums results in failed verification self.assertFalse(ft.verify_checksum(fp, '7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db')) # checksum of length 32 is assumed to be MD5, length 64 to be SHA256, other lengths not allowed @@ -325,25 +319,14 @@ def test_checksums(self): for checksum_type, checksum in broken_checksums.items(): self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum) self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum))) - # md5 is default - self.assertFalse(ft.compute_checksum(fp) == broken_checksums['md5']) - self.assertFalse(ft.verify_checksum(fp, broken_checksums['md5'])) + # sha256 is default + self.assertFalse(ft.compute_checksum(fp) == broken_checksums['sha256']) self.assertFalse(ft.verify_checksum(fp, broken_checksums['sha256'])) # test specify alternative checksums alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', known_checksums['sha256']) self.assertTrue(ft.verify_checksum(fp, alt_checksums)) - alt_checksums = ('fecf50db81148786647312bbd3b5c740', '2c829facaba19c0fcd81f9ce96bef712', - '840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5']) - self.assertTrue(ft.verify_checksum(fp, alt_checksums)) - - alt_checksums = ('840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'], '2c829facaba19c0fcd81f9ce96bef712') - self.assertTrue(ft.verify_checksum(fp, alt_checksums)) - - alt_checksums = (known_checksums['md5'], '840078aeb4b5d69506e7c8edae1e1b89', '2c829facaba19c0fcd81f9ce96bef712') - self.assertTrue(ft.verify_checksum(fp, alt_checksums)) - alt_checksums = (known_checksums['sha256'],) self.assertTrue(ft.verify_checksum(fp, alt_checksums)) @@ -365,16 +348,84 @@ def test_checksums(self): init_config(build_options=build_options) self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, None) - self.assertTrue(ft.verify_checksum(fp, known_checksums['md5'])) self.assertTrue(ft.verify_checksum(fp, known_checksums['sha256'])) # Test dictionary-type checksums - for checksum in [known_checksums[x] for x in ('md5', 'sha256')]: + for checksum in [known_checksums[x] for x in ['sha256']]: + dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'} + self.assertTrue(ft.verify_checksum(fp, dict_checksum)) + del dict_checksum[os.path.basename(fp)] + self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum) + + def test_deprecated_checksums(self): + """Test checksum functionality.""" + + fp = os.path.join(self.test_prefix, 'test.txt') + ft.write_file(fp, "easybuild\n") + + known_checksums = { + 'adler32': '0x379257805', + 'crc32': '0x1457143216', + 'md5': '7167b64b1ca062b9674ffef46f9325db', + 'sha1': 'db05b79e09a4cc67e9dd30b313b5488813db3190', + } + + self.allow_deprecated_behaviour() + self.mock_stderr(True) # just to capture deprecation warning + + # make sure checksums computation/verification is correct + for checksum_type, checksum in known_checksums.items(): + self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum) + self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum))) + + # MD5 checksums can be verified without specifying type + self.assertTrue(ft.verify_checksum(fp, known_checksums['md5'])) + + # providing non-matching MD5 checksums results in failed verification + self.assertFalse(ft.verify_checksum(fp, '1c49562c4b404f3120a3fa0926c8d09c')) + + # checksum of length 32 is assumed to be MD5, length 64 to be SHA256, other lengths not allowed + # checksum of length other than 32/64 yields an error + error_pattern = r"Length of checksum '.*' \(\d+\) does not match with either MD5 \(32\) or SHA256 \(64\)" + for checksum in ['tooshort', 'inbetween32and64charactersisnotgoodeither', known_checksums['md5'] + 'foo']: + self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum) + + # make sure faulty checksums are reported + broken_checksums = {typ: (val[:-3] + 'foo') for typ, val in known_checksums.items()} + for checksum_type, checksum in broken_checksums.items(): + self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum) + self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum))) + self.assertFalse(ft.verify_checksum(fp, broken_checksums['md5'])) + + # test specify alternative checksums + alt_checksums = ('fecf50db81148786647312bbd3b5c740', '2c829facaba19c0fcd81f9ce96bef712', + '840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5']) + self.assertTrue(ft.verify_checksum(fp, alt_checksums)) + + alt_checksums = ('840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'], '2c829facaba19c0fcd81f9ce96bef712') + self.assertTrue(ft.verify_checksum(fp, alt_checksums)) + + alt_checksums = (known_checksums['md5'], '840078aeb4b5d69506e7c8edae1e1b89', '2c829facaba19c0fcd81f9ce96bef712') + self.assertTrue(ft.verify_checksum(fp, alt_checksums)) + + # check whether missing checksums are enforced + build_options = { + 'enforce_checksums': True, + } + init_config(build_options=build_options) + + self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, None) + self.assertTrue(ft.verify_checksum(fp, known_checksums['md5'])) + + # Test dictionary-type checksums + for checksum in [known_checksums[x] for x in ['md5']]: dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'} self.assertTrue(ft.verify_checksum(fp, dict_checksum)) del dict_checksum[os.path.basename(fp)] self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum) + self.mock_stderr(False) + def test_common_path_prefix(self): """Test get common path prefix for a list of paths.""" self.assertEqual(ft.det_common_path_prefix(['/foo/bar/foo', '/foo/bar/baz', '/foo/bar/bar']), '/foo/bar') diff --git a/test/framework/options.py b/test/framework/options.py index c401105116..9bf00bbac6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6187,18 +6187,19 @@ def test_inject_checksums(self): self.assertNotIn('checksums = ', toy_ec_txt) write_file(test_ec, toy_ec_txt) - args = [test_ec, '--inject-checksums=md5'] + args = [test_ec, '--inject-checksums=sha256'] stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) patterns = [ - r"^== injecting md5 checksums in .*/test\.eb$", + r"^== injecting sha256 checksums in .*/test\.eb$", r"^== fetching sources & patches for test\.eb\.\.\.$", r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+\.\.\.$", - r"^== injecting md5 checksums for sources & patches in test\.eb\.\.\.$", - r"^== \* toy-0.0\.tar\.gz: be662daa971a640e40be5c804d9d7d10$", - r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: a99f2a72cee1689a2f7e3ace0356efb1$", - r"^== \* toy-extra\.txt: 3b0787b3bf36603ae1398c4a49097893$", + r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$", + r"^== \* toy-0.0\.tar\.gz: 44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc$", + r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: " # no comma, continues on next line + r"81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487$", + r"^== \* toy-extra\.txt: 4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458$", ] for pattern in patterns: regex = re.compile(pattern, re.M) @@ -6214,9 +6215,10 @@ def test_inject_checksums(self): # no parse errors for updated easyconfig file... ec = EasyConfigParser(test_ec).get_config_dict() checksums = [ - {'toy-0.0.tar.gz': 'be662daa971a640e40be5c804d9d7d10'}, - {'toy-0.0_fix-silly-typo-in-printf-statement.patch': 'a99f2a72cee1689a2f7e3ace0356efb1'}, - {'toy-extra.txt': '3b0787b3bf36603ae1398c4a49097893'}, + {'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'}, + {'toy-0.0_fix-silly-typo-in-printf-statement.patch': + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'}, + {'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'}, ] self.assertEqual(ec['checksums'], checksums) @@ -6479,8 +6481,8 @@ def test_tmp_logdir(self): '--debug', '--tmp-logdir=%s' % tmp_logdir, ] - with self.mocked_stdout_stderr(): - self.eb_main(args, do_build=True, raise_error=True) + # with self.mocked_stdout_stderr(): + self.eb_main(args, do_build=True, raise_error=True) tmp_logs = os.listdir(tmp_logdir) self.assertEqual(len(tmp_logs), 1) From f940ca90b249c0d7a341ac79519678d4e3ba4e44 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 5 May 2024 13:37:16 +0100 Subject: [PATCH 287/430] flake8 --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 60fab80ef5..e8c45758da 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1200,7 +1200,7 @@ def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM): if checksum_type in ['adler32', 'crc32', 'md5', 'sha1', 'size']: _log.deprecated("Checksum type %s is deprecated. Use sha256 (default) or sha512 instead" % checksum_type, - '6.0') + '6.0') try: checksum = CHECKSUM_FUNCTIONS[checksum_type](path) From 33a4318f12d287a8302dd28e0d719ea14cbe1181 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 5 May 2024 13:40:45 +0100 Subject: [PATCH 288/430] revert debugging change --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 9bf00bbac6..1dd1381a51 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6481,8 +6481,8 @@ def test_tmp_logdir(self): '--debug', '--tmp-logdir=%s' % tmp_logdir, ] - # with self.mocked_stdout_stderr(): - self.eb_main(args, do_build=True, raise_error=True) + with self.mocked_stdout_stderr(): + self.eb_main(args, do_build=True, raise_error=True) tmp_logs = os.listdir(tmp_logdir) self.assertEqual(len(tmp_logs), 1) From 7e9e3326f51461822cf7b72170ba044d1dc337e6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 5 May 2024 14:04:18 +0100 Subject: [PATCH 289/430] fix tests --- test/framework/easyblock.py | 7 ++----- .../test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb | 9 +++------ .../easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb | 9 +++------ test/framework/easyconfigs/v2.0/toy-with-sections.eb | 1 - test/framework/easyconfigs/v2.0/toy.eb | 1 - test/framework/filetools.py | 8 ++++---- test/framework/toy_build.py | 2 +- 7 files changed, 13 insertions(+), 24 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index eed6970e50..7c2a8614e9 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1536,9 +1536,8 @@ def test_fetch_sources(self): self.assertTrue(os.path.samefile(eb.src[0]['path'], toy_source)) self.assertEqual(eb.src[0]['name'], 'toy-0.0.tar.gz') self.assertEqual(eb.src[0]['cmd'], None) - self.assertEqual(len(eb.src[0]['checksum']), 7) - self.assertEqual(eb.src[0]['checksum'][0], 'be662daa971a640e40be5c804d9d7d10') - self.assertEqual(eb.src[0]['checksum'][1], '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc') + self.assertEqual(len(eb.src[0]['checksum']), 2) + self.assertEqual(eb.src[0]['checksum'][0], '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc') # reconfigure EasyBuild so we can check 'downloaded' sources os.environ['EASYBUILD_SOURCEPATH'] = self.test_prefix @@ -2566,8 +2565,6 @@ def test_checksum_step(self): copy_file(toy_ec, self.test_prefix) toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec)) ectxt = read_file(toy_ec) - # replace MD5 checksum for toy-0.0.tar.gz - ectxt = ectxt.replace('be662daa971a640e40be5c804d9d7d10', '00112233445566778899aabbccddeeff') # replace SHA256 checksums for source of bar extension ectxt = ectxt.replace('f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7', '01234567' * 8) write_file(toy_ec, ectxt) diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb index ced8241a45..70a231fc8d 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb @@ -10,13 +10,10 @@ toolchainopts = {'pic': True, 'opt': True, 'optarch': True} sources = [SOURCE_TAR_GZ] checksums = [[ - 'be662daa971a640e40be5c804d9d7d10', # default (MD5) '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) - ('adler32', '0x998410035'), - ('crc32', '0x1553842328'), - ('md5', 'be662daa971a640e40be5c804d9d7d10'), - ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), - ('size', 273), + ('sha512', + '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0' + 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'), {SOURCE_TAR_GZ: '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', 'bar.tgz': '33ac60685a3e29538db5094259ea85c15906cbd0f74368733f4111eab6187c8f'}, ]] diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb index 925432d02a..c8d0504764 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb @@ -9,13 +9,10 @@ toolchainopts = {'pic': True, 'opt': True, 'optarch': True} sources = [SOURCE_TAR_GZ] checksums = [[ - 'be662daa971a640e40be5c804d9d7d10', # default (MD5) '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) - ('adler32', '0x998410035'), - ('crc32', '0x1553842328'), - ('md5', 'be662daa971a640e40be5c804d9d7d10'), - ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), - ('size', 273), + ('sha512', + '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0' + 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'), ]] patches = [ 'toy-0.0_fix-silly-typo-in-printf-statement.patch', diff --git a/test/framework/easyconfigs/v2.0/toy-with-sections.eb b/test/framework/easyconfigs/v2.0/toy-with-sections.eb index 34b9af0dcd..a1a508bcf0 100644 --- a/test/framework/easyconfigs/v2.0/toy-with-sections.eb +++ b/test/framework/easyconfigs/v2.0/toy-with-sections.eb @@ -16,7 +16,6 @@ software_license_urls = ['https://github.com/easybuilders/easybuild/wiki/License sources = ['%(name)s-0.0.tar.gz'] # purposely fixed to 0.0 checksums = [ - 'be662daa971a640e40be5c804d9d7d10', # MD5 '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # SHA256 ] diff --git a/test/framework/easyconfigs/v2.0/toy.eb b/test/framework/easyconfigs/v2.0/toy.eb index a1cdfaf6d8..3c5a000f1b 100644 --- a/test/framework/easyconfigs/v2.0/toy.eb +++ b/test/framework/easyconfigs/v2.0/toy.eb @@ -16,7 +16,6 @@ software_license_urls = ['https://github.com/easybuilders/easybuild/wiki/License sources = ['%(name)s-0.0.tar.gz'] # purposely fixed to 0.0 checksums = [ - 'be662daa971a640e40be5c804d9d7d10', # MD5 '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # SHA256 ] diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 636503b9cc..f98a9e7c13 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1213,9 +1213,9 @@ def test_multidiff(self): self.assertTrue(lines[8].startswith(expected)) # no postinstallcmds in toy-0.0-deps.eb - expected = "29 %s+ postinstallcmds = " % green + expected = "26 %s+ postinstallcmds = " % green self.assertTrue(any(line.startswith(expected) for line in lines)) - expected = "30 %s+%s (1/2) toy-0.0" % (green, endcol) + expected = "27 %s+%s (1/2) toy-0.0" % (green, endcol) self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines)) self.assertEqual(lines[-1], "=====") @@ -1234,9 +1234,9 @@ def test_multidiff(self): self.assertTrue(lines[8].startswith(expected)) # no postinstallcmds in toy-0.0-deps.eb - expected = "29 + postinstallcmds = " + expected = "26 + postinstallcmds = " self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines)) - expected = "30 + (1/2) toy-0.0-" + expected = "27 + (1/2) toy-0.0-" self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines)) self.assertEqual(lines[-1], "=====") diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 121bb43367..73e452c8a9 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -262,7 +262,7 @@ def test_toy_broken(self): broken_toy_ec = os.path.join(tmpdir, "toy-broken.eb") toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') broken_toy_ec_txt = read_file(toy_ec_file) - broken_toy_ec_txt += "checksums = ['clearywrongMD5checksumoflength32']" + broken_toy_ec_txt += "checksums = ['clearywrongSHA256checksumoflength64-0123456789012345678901234567']" write_file(broken_toy_ec, broken_toy_ec_txt) error_regex = "Checksum verification .* failed" self.assertErrorRegex(EasyBuildError, error_regex, self.run_test_toy_build_with_output, ec_file=broken_toy_ec, From 78228359eed7c9a562240a6823f6a24d5c3442c4 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 5 May 2024 14:38:52 +0100 Subject: [PATCH 290/430] fix more tests --- test/framework/easyblock.py | 6 +++++- test/framework/easyconfig.py | 4 ++-- .../easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 7c2a8614e9..663809b903 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2565,6 +2565,8 @@ def test_checksum_step(self): copy_file(toy_ec, self.test_prefix) toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec)) ectxt = read_file(toy_ec) + # replace SHA256 checksum for toy-0.0.tar.gz + ectxt = ectxt.replace('44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', '76543210' * 8) # replace SHA256 checksums for source of bar extension ectxt = ectxt.replace('f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7', '01234567' * 8) write_file(toy_ec, ectxt) @@ -2654,7 +2656,9 @@ def test_checksum_step(self): self.mock_stderr(False) self.mock_stdout(False) self.assertEqual(stdout, '') - self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz") + print(stderr.strip()) + self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz\n\n\n" + "WARNING: Ignoring failing checksum verification for toy-0.0.tar.gz") def test_check_checksums(self): """Test for check_checksums_for and check_checksums methods.""" diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 7dc088bff3..fd34622a8e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -485,7 +485,7 @@ def test_exts_list(self): # SHA256 checksum for source (gzip-1.4.eb) "6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45", # SHA256 checksum for 'patch' (toy-0.0.eb) - "2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c", + "177b34bcdfa1abde96f30354848a01894ebc9c24913bc5145306cd30f78fc8ad", ], }), # Can use templates in name and version @@ -509,7 +509,7 @@ def test_exts_list(self): self.assertEqual(exts_sources[1]['version'], '2.0') self.assertEqual(exts_sources[1]['options'], { 'checksums': ['6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45', - '2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c'], + '177b34bcdfa1abde96f30354848a01894ebc9c24913bc5145306cd30f78fc8ad'], 'patches': [('toy-0.0.eb', '.')], 'source_tmpl': 'gzip-1.4.eb', 'source_urls': [('http://example.com', 'suffix')], diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb index 68ece2259f..e02e21f7ae 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb @@ -11,8 +11,8 @@ toolchain = SYSTEM sources = [SOURCE_TAR_GZ] patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] checksums = [ - ('adler32', '0x998410035'), - 'a99f2a72cee1689a2f7e3ace0356efb1', + '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487', ] moduleclass = 'tools' From d905f02b8500f768cc11e0b719382549e1ce80e3 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 5 May 2024 14:46:45 +0100 Subject: [PATCH 291/430] remove debug print --- test/framework/easyblock.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 663809b903..d2d9918599 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2656,7 +2656,6 @@ def test_checksum_step(self): self.mock_stderr(False) self.mock_stdout(False) self.assertEqual(stdout, '') - print(stderr.strip()) self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz\n\n\n" "WARNING: Ignoring failing checksum verification for toy-0.0.tar.gz") From ee2719840d17b7a7c0f198f6ec6abe2d55156abf Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 5 May 2024 16:14:18 +0100 Subject: [PATCH 292/430] install easyblocks for framework tests --- .github/workflows/unit_tests.yml | 12 ++++++++++++ .github/workflows/unit_tests_python2.yml | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 07ac74fe1a..d2c2dc3cbc 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -142,6 +142,18 @@ jobs: export PREFIX=/tmp/$USER/$GITHUB_SHA pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz + - name: install easyblocks + run: | + cd $HOME + # first determine which branch of easybuild-easyblocks repo to install + BRANCH=develop + if [ "x$GITHUB_BASE_REF" = 'xmain' ]; then BRANCH=main; fi + if [ "x$GITHUB_BASE_REF" = 'x4.x' ]; then BRANCH=4.x; fi + echo "Using easybuild-easyblocks branch $BRANCH (\$GITHUB_BASE_REF $GITHUB_BASE_REF)" + git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-easyblocks.git + cd easybuild-easyblocks; git log -n 1; cd - + pip install $PWD/easybuild-easyblocks + - name: run test suite env: EB_VERBOSE: 1 diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 1b921ee83c..252d5f7e6f 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -57,6 +57,18 @@ jobs: export PREFIX=/tmp/$USER/$GITHUB_SHA python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + - name: install easyblocks + run: | + cd $HOME + # first determine which branch of easybuild-easyblocks repo to install + BRANCH=develop + if [ "x$GITHUB_BASE_REF" = 'xmain' ]; then BRANCH=main; fi + if [ "x$GITHUB_BASE_REF" = 'x4.x' ]; then BRANCH=4.x; fi + echo "Using easybuild-easyblocks branch $BRANCH (\$GITHUB_BASE_REF $GITHUB_BASE_REF)" + git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-easyblocks.git + cd easybuild-easyblocks; git log -n 1; cd - + pip install $PWD/easybuild-easyblocks + - name: run test suite run: | # run tests *outside* of checked out easybuild-framework directory, From 18fef0a0d55f3f2d839910b1e3e4a48bfc815b6a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 6 May 2024 12:17:46 +0200 Subject: [PATCH 293/430] define run.CWD_NOTFOUND_ERROR with error message for failures --- easybuild/tools/filetools.py | 11 ++++++----- easybuild/tools/run.py | 6 +++++- test/framework/filetools.py | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d8990560b1..133840efc2 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -66,7 +66,7 @@ from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.hooks import load_source -from easybuild.tools.run import run_shell_cmd +from easybuild.tools.run import CWD_NOTFOUND_ERROR, run_shell_cmd from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -415,10 +415,10 @@ def get_cwd(must_exist=True): cwd = os.getcwd() except FileNotFoundError as err: if must_exist is True: - raise EasyBuildError("Working directory does not exist") - else: - _log.debug("Failed to determine current working directory, but proceeding anyway: %s", err) - cwd = None + raise EasyBuildError(CWD_NOTFOUND_ERROR) + + _log.debug("Failed to determine current working directory, but proceeding anyway: %s", err) + cwd = None return cwd @@ -439,6 +439,7 @@ def change_dir(path): raise EasyBuildError("Failed to change from %s to %s: %s", prev_dir, path, err) # determine final working directory: must exist + # stoplight meant to catch filesystems in a faulty state get_cwd() return prev_dir diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 6a3d29972d..b8196940f4 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -83,6 +83,10 @@ "ulimit -u", # used in det_parallelism ) +CWD_NOTFOUND_ERROR = ( + "Current working directory does not exist! It was either unexpectedly removed " + "by an external process to EasyBuild or the filesystem is misbehaving." +) RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', 'out_file', 'err_file', 'thread_id', 'task_id')) @@ -318,7 +322,7 @@ def to_cmd_str(cmd): try: work_dir = os.getcwd() except FileNotFoundError: - raise EasyBuildError("Working directory does not exist") + raise EasyBuildError(CWD_NOTFOUND_ERROR) cmd_str = to_cmd_str(cmd) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index bff8b70ba9..c6494aeb2b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2231,7 +2231,7 @@ def test_get_cwd(self): self.assertTrue(os.path.samefile(ft.get_cwd(), toy_dir)) os.rmdir(toy_dir) - self.assertErrorRegex(EasyBuildError, "Working directory does not exist", ft.get_cwd) + self.assertErrorRegex(EasyBuildError, ft.CWD_NOTFOUND_ERROR, ft.get_cwd) self.assertEqual(ft.get_cwd(must_exist=False), None) From c3af7a8b04867c045f806fc9d42e19625f137825 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 6 May 2024 12:24:01 +0200 Subject: [PATCH 294/430] fix order of imports from filetools in tools.robot Co-authored-by: Kenneth Hoste --- easybuild/tools/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index a5421b7225..edef824fbc 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -45,7 +45,7 @@ from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import get_cwd, det_common_path_prefix, search_file +from easybuild.tools.filetools import det_common_path_prefix, get_cwd, search_file from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.utilities import flatten, nub From 9e9390458b35cff6a25a5036ab259425acf1e14c Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Mon, 6 May 2024 13:10:36 +0100 Subject: [PATCH 295/430] log a warning, without adding a print statement when no easyblocks version determined --- .github/workflows/unit_tests.yml | 12 ------------ .github/workflows/unit_tests_python2.yml | 12 ------------ easybuild/main.py | 4 +++- 3 files changed, 3 insertions(+), 25 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index d2c2dc3cbc..07ac74fe1a 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -142,18 +142,6 @@ jobs: export PREFIX=/tmp/$USER/$GITHUB_SHA pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz - - name: install easyblocks - run: | - cd $HOME - # first determine which branch of easybuild-easyblocks repo to install - BRANCH=develop - if [ "x$GITHUB_BASE_REF" = 'xmain' ]; then BRANCH=main; fi - if [ "x$GITHUB_BASE_REF" = 'x4.x' ]; then BRANCH=4.x; fi - echo "Using easybuild-easyblocks branch $BRANCH (\$GITHUB_BASE_REF $GITHUB_BASE_REF)" - git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-easyblocks.git - cd easybuild-easyblocks; git log -n 1; cd - - pip install $PWD/easybuild-easyblocks - - name: run test suite env: EB_VERBOSE: 1 diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 252d5f7e6f..1b921ee83c 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -57,18 +57,6 @@ jobs: export PREFIX=/tmp/$USER/$GITHUB_SHA python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz - - name: install easyblocks - run: | - cd $HOME - # first determine which branch of easybuild-easyblocks repo to install - BRANCH=develop - if [ "x$GITHUB_BASE_REF" = 'xmain' ]; then BRANCH=main; fi - if [ "x$GITHUB_BASE_REF" = 'x4.x' ]; then BRANCH=4.x; fi - echo "Using easybuild-easyblocks branch $BRANCH (\$GITHUB_BASE_REF $GITHUB_BASE_REF)" - git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-easyblocks.git - cd easybuild-easyblocks; git log -n 1; cd - - pip install $PWD/easybuild-easyblocks - - name: run test suite run: | # run tests *outside* of checked out easybuild-framework directory, diff --git a/easybuild/main.py b/easybuild/main.py index 00b2899227..00bc362666 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -621,7 +621,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr # compare running Framework and EasyBlocks versions if EASYBLOCKS_VERSION == UNKNOWN_VERSION: - print_warning("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework") + # most likely reason is running framework unit tests with no easyblocks installation + # so log a warning, to avoid test related issues + _log.warning("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework") elif different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION): raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION, EASYBLOCKS_VERSION)) From 1ec53a50c6ab92e671c51629de87d63218b6f3b7 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Mon, 6 May 2024 13:59:27 +0100 Subject: [PATCH 296/430] version check on from pr --- easybuild/tools/github.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 9a70109c00..5691be973c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -60,6 +60,7 @@ from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen from easybuild.tools.systemtools import UNKNOWN, get_tool_version from easybuild.tools.utilities import nub, only_if_module_is_available +from easybuild.tools.version import FRAMEWORK_VERSION, different_major_versions _log = fancylogger.getLogger('github', fname=False) @@ -588,13 +589,38 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi raise EasyBuildError("Couldn't find path to patched file %s", full_path) if github_repo == GITHUB_EASYCONFIGS_REPO: - print('get ecs version', os.path.join(final_path, 'setup.py')) + ver = _get_version_for_repo(os.path.join(final_path, 'setup.py')) elif github_repo == GITHUB_EASYBLOCKS_REPO: - print('get blocks version', sys.path, final_path) + ver = _get_version_for_repo(os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py')) + + if different_major_versions(FRAMEWORK_VERSION, ver): + raise EasyBuildError("Framework (%s) is a different major version than PR target (%s)." % (FRAMEWORK_VERSION, + ver)) return files +def _get_version_for_repo(filename): + """Extract version from filename.""" + _log.debug("Extract version from %s" % filename) + + try: + ver_line = "" + with open(filename) as f: + for line in f.readlines(): + if line.startswith("VERSION "): + ver_line = line + break + + # version can be a string or LooseVersion + res = re.search(r"""^VERSION = .*['"](.*)['"].?$""", ver_line) + + _log.debug("PR target version is %s" % res.group(1)) + return res.group(1) + except: + raise EasyBuildError("Couldn't determine version of PR from %s" % filename) + + def fetch_easyblocks_from_pr(pr, path=None, github_user=None): """Fetch patched easyblocks for a particular PR.""" return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) From 0b18d9607401dc8a80a222fc12cc883bb99d18a9 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Mon, 6 May 2024 14:01:44 +0100 Subject: [PATCH 297/430] flake8 --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 5691be973c..0b5a2ff171 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -617,7 +617,7 @@ def _get_version_for_repo(filename): _log.debug("PR target version is %s" % res.group(1)) return res.group(1) - except: + except Exception: raise EasyBuildError("Couldn't determine version of PR from %s" % filename) From db03aa7e0eefb940b08820880bd94c196da98402 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Sun, 26 Nov 2023 16:19:15 +0100 Subject: [PATCH 298/430] return to original directory earlier after executing a command --- easybuild/tools/run.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index ca33a94a9c..99b3252daf 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -505,6 +505,12 @@ def to_cmd_str(cmd): if fail_on_error: raise_run_shell_cmd_error(res) + # return to original work dir after command execution + try: + os.chdir(res.work_dir) + except OSError as err: + raise EasyBuildError(f"Failed to return to {res.work_dir} after executing command `{cmd_str}`: {err}") + if with_hooks: run_hook_kwargs = { 'exit_code': res.exit_code, From c462a05611696f3d272d1811585247c0e682f849 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Dec 2023 09:11:10 +0100 Subject: [PATCH 299/430] enhance test_run_cmd_with_hooks to check whether run_cmd/complete_cmd is robust against command that removes the directory it's running in --- test/framework/run.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index 23fdb562d3..a5d40ad5a7 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -24,7 +24,7 @@ # along with EasyBuild. If not, see . # # """ -Unit tests for filetools.py +Unit tests for run.py @author: Toon Willems (Ghent University) @author: Kenneth Hoste (Ghent University) @@ -1854,6 +1854,22 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ]) self.assertEqual(stdout, expected_stdout) + # also check with a command that destroys the directory it's running in + workdir = os.path.join(self.test_prefix, 'workdir') + mkdir(workdir) + + with self.mocked_stdout_stderr(): + run_shell_cmd("pwd; rm -rf %(workdir)s; echo '%(workdir)s removed'" % {'workdir': workdir}, path=workdir) + stdout = self.get_stdout() + + patterns = [ + r"pre-run hook 'pwd; rm .*/workdir removed'", + r"post-run hook 'pwd;\s*rm.*/workdir removed'' \(exit code: 0, output: '.*\n.*/workdir removed\n'\)", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def suite(): """ returns all the testcases in this module """ From 9f096cbf05e68d10c5af57a0471837b24f49bef7 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 6 May 2024 17:00:11 +0200 Subject: [PATCH 300/430] test run_shell_cmd with hooks enabled with commands that remove the original workdir or jump into a new working subdirectory and remove it --- test/framework/run.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index a5d40ad5a7..685afcad9a 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1848,26 +1848,39 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): stdout = self.get_stdout() expected_stdout = '\n'.join([ - "pre-run hook 'make' in %s" % cwd, + f"pre-run hook 'make' in {cwd}", "post-run hook 'echo make' (exit code: 0, output: 'make\n')", '', ]) self.assertEqual(stdout, expected_stdout) - # also check with a command that destroys the directory it's running in + # check commands that destroy directories inside initial working directory + # 1. test destruction of main working directory workdir = os.path.join(self.test_prefix, 'workdir') mkdir(workdir) + with self.mocked_stdout_stderr(): + cmd_workdir_rm = "echo 'Command that ends up removing working directory' && pwd && " + cmd_workdir_rm += f"rm -rf {workdir} && echo 'Working directory removed.'" + error_pattern = rf"Failed to return to {workdir} after executing command" + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir) + # 2. test destruction of current working (sub)directory inside original working directory + sub_workdir = os.path.join(self.test_prefix, 'workdir', 'subworkdir') + mkdir(sub_workdir, parents=True) with self.mocked_stdout_stderr(): - run_shell_cmd("pwd; rm -rf %(workdir)s; echo '%(workdir)s removed'" % {'workdir': workdir}, path=workdir) + cmd_workdir_rm = "echo 'Command that jumps to working subdir and removes it' && " + cmd_workdir_rm += f"cd {sub_workdir} && pwd && rm -rf {sub_workdir} && " + cmd_workdir_rm += "echo 'Working directory removed.'" + run_shell_cmd(cmd_workdir_rm, work_dir=workdir) stdout = self.get_stdout() - patterns = [ - r"pre-run hook 'pwd; rm .*/workdir removed'", - r"post-run hook 'pwd;\s*rm.*/workdir removed'' \(exit code: 0, output: '.*\n.*/workdir removed\n'\)", + expected_stdout_patterns = [ + rf"pre-run hook '{cmd_workdir_rm}' in {workdir}", + (rf"post-run hook '{cmd_workdir_rm}' \(exit code: 0, " + rf"output: 'Command that jumps.*\n{sub_workdir}\nWorking directory removed\..*'\)"), ] - for pattern in patterns: - regex = re.compile(pattern, re.M) + for pattern in expected_stdout_patterns: + regex = re.compile(pattern, re.S) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) From 0a3091cb025b12674bf52b218b53d2384f948976 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 6 May 2024 17:43:11 +0200 Subject: [PATCH 301/430] update test_run_shell_cmd_work_dir to consider run_shell_cmd with and without work_dir set --- test/framework/run.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 685afcad9a..aab8d3d3ec 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -500,24 +500,40 @@ def test_run_shell_cmd_work_dir(self): """ Test running shell command in specific directory with run_shell_cmd function. """ - orig_wd = os.getcwd() - self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) - test_dir = os.path.join(self.test_prefix, 'test') + test_workdir = os.path.join(self.test_prefix, 'test', 'workdir') for fn in ('foo.txt', 'bar.txt'): - write_file(os.path.join(test_dir, fn), 'test') + write_file(os.path.join(test_workdir, fn), 'test') + + os.chdir(test_dir) + orig_wd = os.getcwd() + self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) cmd = "ls | sort" + + # undefined working directory + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd) + + self.assertEqual(res.cmd, cmd) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, 'workdir\n') + self.assertEqual(res.stderr, None) + self.assertEqual(res.work_dir, orig_wd) + + self.assertTrue(os.path.samefile(orig_wd, os.getcwd())) + + # defined working directory with self.mocked_stdout_stderr(): - res = run_shell_cmd(cmd, work_dir=test_dir) + res = run_shell_cmd(cmd, work_dir=test_workdir) self.assertEqual(res.cmd, cmd) self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, 'bar.txt\nfoo.txt\n') self.assertEqual(res.stderr, None) - self.assertEqual(res.work_dir, test_dir) + self.assertEqual(res.work_dir, test_workdir) - self.assertTrue(os.path.samefile(orig_wd, os.getcwd())) + self.assertTrue(os.path.samefile(test_workdir, os.getcwd())) def test_run_cmd_log_output(self): """Test run_cmd with log_output enabled""" From 76e38ae40c3fbac4ad2a9789f0ec14ddf1bb5b85 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 10 May 2024 12:07:31 +0200 Subject: [PATCH 302/430] only change directory if undefined CWD as safeguard after command execution in run_shell_cmd --- easybuild/tools/run.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 99b3252daf..0ec3e4d83f 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -505,11 +505,20 @@ def to_cmd_str(cmd): if fail_on_error: raise_run_shell_cmd_error(res) - # return to original work dir after command execution + # check that we still are in a sane environment after command execution + # safeguard against commands that leave behind the system in a borked state try: - os.chdir(res.work_dir) - except OSError as err: - raise EasyBuildError(f"Failed to return to {res.work_dir} after executing command `{cmd_str}`: {err}") + os.getcwd() + except FileNotFoundError: + try: + warn_msg = ( + f"Shell command `{cmd_str}` completed successfully but left system in a broken state. " + f"Changing back to initial working directory: {res.work_dir}" + ) + _log.warning(warn_msg) + os.chdir(res.work_dir) + except OSError as err: + raise EasyBuildError(f"Failed to return to {res.work_dir} after executing command `{cmd_str}`: {err}") if with_hooks: run_hook_kwargs = { From b7db884783dd303a1a3f0b781894057669d9e569 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 10 May 2024 12:08:46 +0200 Subject: [PATCH 303/430] add tests to run_shell_cmd_with_hooks mocking os.getcwd to raise error --- test/framework/run.py | 71 +++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index aab8d3d3ec..f950f3cab5 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -32,6 +32,7 @@ """ import contextlib import glob +import mock import os import re import signal @@ -511,7 +512,7 @@ def test_run_shell_cmd_work_dir(self): cmd = "ls | sort" - # undefined working directory + # working directory is not explicitly defined with self.mocked_stdout_stderr(): res = run_shell_cmd(cmd) @@ -523,7 +524,7 @@ def test_run_shell_cmd_work_dir(self): self.assertTrue(os.path.samefile(orig_wd, os.getcwd())) - # defined working directory + # working directory is explicitly defined with self.mocked_stdout_stderr(): res = run_shell_cmd(cmd, work_dir=test_workdir) @@ -533,7 +534,7 @@ def test_run_shell_cmd_work_dir(self): self.assertEqual(res.stderr, None) self.assertEqual(res.work_dir, test_workdir) - self.assertTrue(os.path.samefile(test_workdir, os.getcwd())) + self.assertTrue(os.path.samefile(orig_wd, os.getcwd())) def test_run_cmd_log_output(self): """Test run_cmd with log_output enabled""" @@ -1870,34 +1871,64 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ]) self.assertEqual(stdout, expected_stdout) - # check commands that destroy directories inside initial working directory - # 1. test destruction of main working directory + # Test commands that destroy directories inside initial working directory workdir = os.path.join(self.test_prefix, 'workdir') - mkdir(workdir) - with self.mocked_stdout_stderr(): - cmd_workdir_rm = "echo 'Command that ends up removing working directory' && pwd && " - cmd_workdir_rm += f"rm -rf {workdir} && echo 'Working directory removed.'" - error_pattern = rf"Failed to return to {workdir} after executing command" - self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir) + sub_workdir = os.path.join(workdir, 'subworkdir') + + # 1. test destruction of CWD which is a subdirectory inside original working directory + cmd_workdir_rm = "echo 'Command that jumps to subdir and removes it' && " + cmd_workdir_rm += f"cd {sub_workdir} && pwd && rm -rf {sub_workdir} && " + cmd_workdir_rm += "echo 'Working sub-directory removed.'" + expected_stdout_patterns = [ + rf"pre-run hook '{cmd_workdir_rm}' in {workdir}", + (rf"post-run hook '{cmd_workdir_rm}' \(exit code: 0, " + rf"output: 'Command that jumps to subdir.*\n{sub_workdir}\nWorking sub-directory removed\..*'\)"), + ] + expected_stdout_patterns = [re.compile(pattern, re.S) for pattern in expected_stdout_patterns] - # 2. test destruction of current working (sub)directory inside original working directory - sub_workdir = os.path.join(self.test_prefix, 'workdir', 'subworkdir') + # in a robust system mkdir(sub_workdir, parents=True) with self.mocked_stdout_stderr(): - cmd_workdir_rm = "echo 'Command that jumps to working subdir and removes it' && " - cmd_workdir_rm += f"cd {sub_workdir} && pwd && rm -rf {sub_workdir} && " - cmd_workdir_rm += "echo 'Working directory removed.'" run_shell_cmd(cmd_workdir_rm, work_dir=workdir) stdout = self.get_stdout() + for regex in expected_stdout_patterns: + self.assertTrue(regex.search(stdout), f"Pattern '{regex.pattern}' should be found in: {stdout}") + + # in a flaky system that ends up in a broken environment after execution + mkdir(sub_workdir, parents=True) + with self.mocked_stdout_stderr(): + with mock.patch('os.getcwd') as mock_getcwd: + mock_getcwd.side_effect = FileNotFoundError() + run_shell_cmd(cmd_workdir_rm, work_dir=workdir) + stdout = self.get_stdout() + + for regex in expected_stdout_patterns: + self.assertTrue(regex.search(stdout), f"Pattern '{regex.pattern}' should be found in: {stdout}") + + # 2. test destruction of CWD which is main working directory passed to run_shell_cmd + cmd_workdir_rm = "echo 'Command that removes working directory' && pwd && " + cmd_workdir_rm += f"rm -rf {workdir} && echo 'Working directory removed.'" expected_stdout_patterns = [ rf"pre-run hook '{cmd_workdir_rm}' in {workdir}", (rf"post-run hook '{cmd_workdir_rm}' \(exit code: 0, " - rf"output: 'Command that jumps.*\n{sub_workdir}\nWorking directory removed\..*'\)"), + rf"output: 'Command that removes working.*\n{workdir}\nWorking directory removed\..*'\)"), ] - for pattern in expected_stdout_patterns: - regex = re.compile(pattern, re.S) - self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + expected_stdout_patterns = [re.compile(pattern, re.S) for pattern in expected_stdout_patterns] + + # in a robust system + mkdir(workdir) + with self.mocked_stdout_stderr(): + error_pattern = rf"Failed to return to {workdir} after executing command" + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir) + + # in a flaky system that ends up in a broken environment after execution + mkdir(workdir) + with self.mocked_stdout_stderr(): + with mock.patch('os.getcwd') as mock_getcwd: + mock_getcwd.side_effect = FileNotFoundError() + error_pattern = rf"Failed to return to {workdir} after executing command" + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir) def suite(): From 98a49daf3ad09694d27a65bdaff2e34ade1decf6 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 10 May 2024 12:22:32 +0200 Subject: [PATCH 304/430] replace mock module with unittest.mock --- test/framework/run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index f950f3cab5..a102fc957a 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -32,7 +32,6 @@ """ import contextlib import glob -import mock import os import re import signal @@ -45,7 +44,7 @@ import time from concurrent.futures import ThreadPoolExecutor from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config -from unittest import TextTestRunner +from unittest import TextTestRunner, mock from easybuild.base.fancylogger import setLogLevelDebug import easybuild.tools.asyncprocess as asyncprocess From 672f6b1d5b71d25a89f13938c2c2f4ed4a546fcb Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 10 May 2024 14:33:31 +0200 Subject: [PATCH 305/430] remove redundant test from test_run_shell_cmd_with_hooks --- test/framework/run.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index a102fc957a..f7df40156b 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1885,7 +1885,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ] expected_stdout_patterns = [re.compile(pattern, re.S) for pattern in expected_stdout_patterns] - # in a robust system + # 1.a. in a robust system mkdir(sub_workdir, parents=True) with self.mocked_stdout_stderr(): run_shell_cmd(cmd_workdir_rm, work_dir=workdir) @@ -1894,7 +1894,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): for regex in expected_stdout_patterns: self.assertTrue(regex.search(stdout), f"Pattern '{regex.pattern}' should be found in: {stdout}") - # in a flaky system that ends up in a broken environment after execution + # 1.b. in a flaky system that ends up in a broken environment after execution mkdir(sub_workdir, parents=True) with self.mocked_stdout_stderr(): with mock.patch('os.getcwd') as mock_getcwd: @@ -1915,20 +1915,11 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ] expected_stdout_patterns = [re.compile(pattern, re.S) for pattern in expected_stdout_patterns] - # in a robust system mkdir(workdir) with self.mocked_stdout_stderr(): error_pattern = rf"Failed to return to {workdir} after executing command" self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir) - # in a flaky system that ends up in a broken environment after execution - mkdir(workdir) - with self.mocked_stdout_stderr(): - with mock.patch('os.getcwd') as mock_getcwd: - mock_getcwd.side_effect = FileNotFoundError() - error_pattern = rf"Failed to return to {workdir} after executing command" - self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir) - def suite(): """ returns all the testcases in this module """ From 7553dcdd4dc1c21923356043daadf684282924b9 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 10 May 2024 18:29:44 +0100 Subject: [PATCH 306/430] do not run unit tests on Python 3.5 --- .github/workflows/eb_command.yml | 2 +- .github/workflows/linting.yml | 2 +- .github/workflows/unit_tests.yml | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index ec4907d28c..220258cd9f 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index d722761208..e7c25b22d7 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 07ac74fe1a..c9f42891ec 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -37,8 +37,6 @@ jobs: lc_all: [""] include: # Test different Python 3 versions with Lmod 8.x - - python: 3.5 - modules_tool: ${{needs.setup.outputs.lmod8}} - python: 3.7 modules_tool: ${{needs.setup.outputs.lmod8}} - python: 3.8 From 31b1846b100ee217765d69a7abdc1826fe0fbe1a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 May 2024 15:29:08 +0200 Subject: [PATCH 307/430] Add tests for (un)escaping when resolving templates --- test/framework/easyconfig.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 45e2852221..c1bbe0cf8d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3654,6 +3654,23 @@ def test_resolve_template(self): # '%(name)' is not a correct template spec (missing trailing 's') self.assertEqual(resolve_template('%(name)', tmpl_dict), '%(name)') + # Correct (un)escaping + values = ( + ('10%', '10%'), + ('%of', '%of'), + ('10%of', '10%of'), + ('%s', '%s'), + ('%%(name)s', '%(name)s'), + ('%%%(name)s', '%FooBar'), + ('%%%%(name)s', '%%(name)s'), + ) + for value, expected in values: + self.assertEqual(resolve_template(value, tmpl_dict), expected) + # Templates are resolved + value += ' %(name)s' + expected += ' FooBar' + self.assertEqual(resolve_template(value, tmpl_dict), expected) + def test_det_subtoolchain_version(self): """Test det_subtoolchain_version function""" _, all_tc_classes = search_toolchain('') From 7323bae811b9aa508049e92bc1456a030c1e38ee Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 May 2024 15:37:22 +0200 Subject: [PATCH 308/430] Fix resolved values in case of failure When the value cannot be resolved usually the original value is returned. However if it contains `%` signs there will be an escape step and that escaped value is returned. That makes it impossible to have values which can only later be resolved, like `cd %(startdir)s` in extensions which would become `cd %%(startdir)s` on the first resolve-attempt. --- easybuild/framework/easyconfig/easyconfig.py | 2 ++ test/framework/easyconfig.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 73182da7ac..2b6aaa751a 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -2026,12 +2026,14 @@ def resolve_template(value, tmpl_dict): # '%(name)s' -> '%(name)s' # '%%(name)s' -> '%%(name)s' if '%' in value: + orig_value = value value = re.sub(re.compile(r'(%)(?!%*\(\w+\)s)'), r'\1\1', value) try: value = value % tmpl_dict except KeyError: _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict) + value = orig_value # Undo "%"-escaping else: # this block deals with references to objects and returns other references # for reading this is ok, but for self['x'] = {} diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c1bbe0cf8d..4b63dc605b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3663,6 +3663,9 @@ def test_resolve_template(self): ('%%(name)s', '%(name)s'), ('%%%(name)s', '%FooBar'), ('%%%%(name)s', '%%(name)s'), + # It doesn't matter what is resolved + ('%%(invalid)s', '%(invalid)s'), + ('%%%%(invalid)s', '%%(invalid)s'), ) for value, expected in values: self.assertEqual(resolve_template(value, tmpl_dict), expected) @@ -3671,6 +3674,10 @@ def test_resolve_template(self): expected += ' FooBar' self.assertEqual(resolve_template(value, tmpl_dict), expected) + # On unknown values the value is returned unchanged + for value in ('%(invalid)s', '%(name)s %(invalid)s', '%%%(invalid)s', '% %(invalid)s', '%s %(invalid)s'): + self.assertEqual(resolve_template(value, tmpl_dict), value) + def test_det_subtoolchain_version(self): """Test det_subtoolchain_version function""" _, all_tc_classes = search_toolchain('') From 23e5b391bbc9fc42c5de3984f3729c500832a335 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 May 2024 15:09:42 +0200 Subject: [PATCH 309/430] Use dict.items() instead of repeatedly getting the value Faster and shorter --- easybuild/framework/easyblock.py | 20 +++++++++---------- easybuild/framework/easyconfig/easyconfig.py | 10 +++++----- easybuild/tools/config.py | 6 +++--- easybuild/tools/docs.py | 4 ++-- easybuild/tools/environment.py | 4 ++-- easybuild/tools/github.py | 4 ++-- .../module_naming_scheme/hierarchical_mns.py | 4 ++-- easybuild/tools/multidiff.py | 4 ++-- easybuild/tools/options.py | 4 ++-- easybuild/tools/systemtools.py | 8 ++++---- easybuild/tools/testing.py | 4 ++-- easybuild/tools/toolchain/toolchain.py | 4 ++-- test/framework/easyconfig.py | 16 +++++++-------- 13 files changed, 46 insertions(+), 46 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 98d71475b0..670966b05d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2197,9 +2197,9 @@ def handle_iterate_opts(self): self.log.info("Current iteration index: %s", self.iter_idx) # pop first element from all iterative easyconfig parameters as next value to use - for opt in self.iter_opts: - if len(self.iter_opts[opt]) > self.iter_idx: - self.cfg[opt] = self.iter_opts[opt][self.iter_idx] + for opt, value in self.iter_opts.items(): + if len(value) > self.iter_idx: + self.cfg[opt] = value[self.iter_idx] else: self.cfg[opt] = '' # empty list => empty option as next value self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) @@ -2211,12 +2211,12 @@ def post_iter_step(self): """Restore options that were iterated over""" # disable templating, since we're messing about with values in self.cfg with self.cfg.disable_templating(): - for opt in self.iter_opts: - self.cfg[opt] = self.iter_opts[opt] + for opt, value in self.iter_opts.items(): + self.cfg[opt] = value # also need to take into account extensions, since those were iterated over as well for ext in self.ext_instances: - ext.cfg[opt] = self.iter_opts[opt] + ext.cfg[opt] = value self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) @@ -4661,14 +4661,14 @@ def inject_checksums_to_json(ecs, checksum_type): # actually inject new checksums or overwrite existing ones (if --force) existing_checksums = app.get_checksums_from_json(always_read=True) - for filename in checksums: + for filename, checksum in checksums.items(): if filename not in existing_checksums: - existing_checksums[filename] = checksums[filename] + existing_checksums[filename] = checksum # don't do anything if the checksum already exist and is the same - elif checksums[filename] != existing_checksums[filename]: + elif checksum != existing_checksums[filename]: if build_option('force'): print_warning("Found existing checksums for %s, overwriting them (due to --force)..." % ec_fn) - existing_checksums[filename] = checksums[filename] + existing_checksums[filename] = checksum else: raise EasyBuildError("Found existing checksum for %s, use --force to overwrite them" % filename) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 5f8170b748..9177338454 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -827,7 +827,7 @@ def check_deprecated(self, path): if depr_msgs: depr_msg = ', '.join(depr_msgs) - depr_maj_ver = int(str(VERSION).split('.')[0]) + 1 + depr_maj_ver = int(str(VERSION).split('.', maxsplit=1)[0]) + 1 depr_ver = '%s.0' % depr_maj_ver more_info_depr_ec = " (see also https://docs.easybuild.io/deprecated-easyconfigs)" @@ -842,8 +842,8 @@ def validate(self, check_osdeps=True): - check license """ self.log.info("Validating easyconfig") - for attr in self.validations: - self._validate(attr, self.validations[attr]) + for attr, valid_values in self.validations.items(): + self._validate(attr, valid_values) if check_osdeps: self.log.info("Checking OS dependencies") @@ -1207,8 +1207,8 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals # templated values should be dumped unresolved with self.disable_templating(): # build dict of default values - default_values = {key: DEFAULT_CONFIG[key][0] for key in DEFAULT_CONFIG} - default_values.update({key: self.extra_options[key][0] for key in self.extra_options}) + default_values = {key: value[0] for key, value in DEFAULT_CONFIG.items()} + default_values.update({key: value[0] for key, value in self.extra_options.items()}) self.generate_template_values() templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS} diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 8dea4e4f68..1864ec0dc0 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -592,12 +592,12 @@ def init_build_options(build_options=None, cmdline_options=None): # seed in defaults to make sure all build options are defined, and that build_option() doesn't fail on valid keys bo = {} for build_options_by_default in [BUILD_OPTIONS_CMDLINE, BUILD_OPTIONS_OTHER]: - for default in build_options_by_default: + for default, options in build_options_by_default.items(): if default == EMPTY_LIST: - for opt in build_options_by_default[default]: + for opt in options: bo[opt] = [] else: - bo.update({opt: default for opt in build_options_by_default[default]}) + bo.update({opt: default for opt in options}) bo.update(active_build_options) # BuildOptions is a singleton, so any future calls to BuildOptions will yield the same instance diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 42d88616f0..ba57976f90 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -807,8 +807,8 @@ def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False # rebuild software, only retain entries with a corresponding available module software, all_software = {}, software - for key in all_software: - for entry in all_software[key]: + for key, entries in all_software.items(): + for entry in entries: if entry['mod_name'] in avail_mod_names: software.setdefault(key, []).append(entry) diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index a5cc708ce2..07f9ef6e50 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -54,8 +54,8 @@ def write_changes(filename): """ try: with open(filename, 'w') as script: - for key in _changes: - script.write('export %s=%s\n' % (key, shell_quote(_changes[key]))) + for key, changed_value in _changes.items(): + script.write('export %s=%s\n' % (key, shell_quote(changed_value))) except IOError as err: raise EasyBuildError("Failed to write to %s: %s", filename, err) reset_changes() diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d1b93c3ad7..b1a8b69260 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1029,8 +1029,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # only consider new easyconfig files for dependencies (not updated ones) for idx in range(len(all_dep_info['ecs'])): if all_dep_info['new'][idx]: - for key in dep_info: - dep_info[key].append(all_dep_info[key][idx]) + for key, values in dep_info.items(): + values.append(all_dep_info[key][idx]) # checkout target branch if pr_branch is None: diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index d0e4ea5966..7b368e4cac 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -204,10 +204,10 @@ def det_modpath_extensions(self, ec): comp_name_ver = None if ec['name'] in extend_comps: - for key in COMP_NAME_VERSION_TEMPLATES: + for key, comp_tmpl in COMP_NAME_VERSION_TEMPLATES.items(): comp_names = key.split(',') if ec['name'] in comp_names: - comp_name, comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key] + comp_name, comp_ver_tmpl = comp_tmpl comp_versions = {ec['name']: self.det_full_version(ec)} if ec['name'] == 'ifort': # 'icc' key should be provided since it's the only one used in the template diff --git a/easybuild/tools/multidiff.py b/easybuild/tools/multidiff.py index 99ee949e56..b8a135bf2f 100644 --- a/easybuild/tools/multidiff.py +++ b/easybuild/tools/multidiff.py @@ -291,8 +291,8 @@ def multidiff(base, files, colored=True): offset -= 1 # construct the multi-diff based on the constructed dict - for line_no in local_diff: - for (line, filename) in local_diff[line_no]: + for line_no, line_infos in local_diff.items(): + for (line, filename) in line_infos: mdiff.parse_line(line_no, line.rstrip(), filename, squigly_dict.get(line, '').rstrip()) return str(mdiff) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index f5d5c801a0..aa134fa706 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1365,9 +1365,9 @@ def show_system_info(self): '', "* GPU:", ]) - for vendor in gpu_info: + for vendor, vendor_gpu in gpu_info.items(): lines.append(" -> %s" % vendor) - for gpu, num in gpu_info[vendor].items(): + for gpu, num in vendor_gpu.items(): lines.append(" -> %sx %s" % (num, gpu)) lines.extend([ diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 73cba6f91a..b4e1c254da 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1367,9 +1367,9 @@ def extract_version(tool): python_version = extract_version(sys.executable) opt_dep_versions = {} - for key in EASYBUILD_OPTIONAL_DEPENDENCIES: + for key, opt_dep in EASYBUILD_OPTIONAL_DEPENDENCIES.items(): - pkg = EASYBUILD_OPTIONAL_DEPENDENCIES[key][0] + pkg = opt_dep[0] if pkg is None: pkg = key.lower() @@ -1395,8 +1395,8 @@ def extract_version(tool): opt_deps_key = "Optional dependencies" checks_data[opt_deps_key] = {} - for key in opt_dep_versions: - checks_data[opt_deps_key][key] = (opt_dep_versions[key], EASYBUILD_OPTIONAL_DEPENDENCIES[key][1]) + for key, version in opt_dep_versions.items(): + checks_data[opt_deps_key][key] = (version, EASYBUILD_OPTIONAL_DEPENDENCIES[key][1]) sys_tools_key = "System tools" checks_data[sys_tools_key] = {} diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 881788337e..dae8a51449 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -321,8 +321,8 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state, gpu_info = get_gpu_info() gpu_str = "" if gpu_info: - for vendor in gpu_info: - for gpu, num in gpu_info[vendor].items(): + for vendor, vendor_gpu in gpu_info.items(): + for gpu, num in vendor_gpu.items(): gpu_str += ", %s x %s %s" % (num, vendor, gpu) os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index c11fb309d3..17916de519 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -588,8 +588,8 @@ def _simulated_load_dependency_module(self, name, version, metadata, verbose=Fal self.log.debug("Defining $EB* environment variables for software named %s", name) env_vars = env_vars_external_module(name, version, metadata) - for key in env_vars: - setvar(key, env_vars[key], verbose=verbose) + for var, value in env_vars.items(): + setvar(var, value, verbose=verbose) def _load_toolchain_module(self, silent=False): """Load toolchain module.""" diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index fd34622a8e..274c8deb05 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2168,8 +2168,8 @@ def test_external_dependencies_templates(self): 'pyshortver': '3.6', 'pyver': '3.6.5', } - for key in expected_template_values: - self.assertEqual(ec.template_values[key], expected_template_values[key]) + for key, expected in expected_template_values.items(): + self.assertEqual(ec.template_values[key], expected) self.assertEqual(ec['versionsuffix'], '-Python-3.6.5-Perl-5.30') @@ -2245,8 +2245,8 @@ def test_quote_str(self): 'foo\\bar': '"foo\\bar"', } - for t in teststrings: - self.assertEqual(quote_str(t), teststrings[t]) + for t, expected in teststrings.items(): + self.assertEqual(quote_str(t), expected) # test escape_newline self.assertEqual(quote_str("foo\nbar", escape_newline=False), '"foo\nbar"') @@ -4815,8 +4815,8 @@ def test_get_cuda_cc_template_value(self): update_build_option('cuda_compute_capabilities', ['6.5', '7.0']) ec = EasyConfig(self.eb_file) - for key in cuda_template_values: - self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key]) + for key, expected in cuda_template_values.items(): + self.assertEqual(ec.get_cuda_cc_template_value(key), expected) update_build_option('cuda_compute_capabilities', None) ec = EasyConfig(self.eb_file) @@ -4828,8 +4828,8 @@ def test_get_cuda_cc_template_value(self): self.prep() ec = EasyConfig(self.eb_file) - for key in cuda_template_values: - self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key]) + for key, expected in cuda_template_values.items(): + self.assertEqual(ec.get_cuda_cc_template_value(key), expected) def test_count_files(self): """Tests for EasyConfig.count_files method.""" From 0319c02fa996322fa6ef9c6cb8397dedb5f431b7 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Mon, 20 May 2024 15:44:05 +0200 Subject: [PATCH 310/430] add support for parameter module_only --- easybuild/framework/easyblock.py | 10 +++++----- easybuild/framework/easyconfig/default.py | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 33110c61dd..9dac204fdb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1068,7 +1068,7 @@ def make_builddir(self): self.log.info("Overriding 'cleanupoldinstall' (to False), 'cleanupoldbuild' (to True) " "and 'keeppreviousinstall' because we're building in the installation directory.") # force cleanup before installation - if build_option('module_only'): + if build_option('module_only') or self.cfg['module_only']: self.log.debug("Disabling cleanupoldbuild because we run as module-only") self.cfg['cleanupoldbuild'] = False else: @@ -1139,7 +1139,7 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): if self.cfg['keeppreviousinstall']: self.log.info("Keeping old directory %s (hopefully you know what you are doing)", dir_name) return - elif build_option('module_only'): + elif build_option('module_only') or self.cfg['module_only']: self.log.info("Not touching existing directory %s in module-only mode...", dir_name) elif clean: remove_dir(dir_name) @@ -2114,7 +2114,7 @@ def guess_start_dir(self): start_dir = '' # do not use the specified 'start_dir' when running as --module-only as # the directory will not exist (extract_step is skipped) - if self.start_dir and not build_option('module_only'): + if self.start_dir and not build_option('module_only') and not self.cfg['module_only']: start_dir = self.start_dir if not os.path.isabs(start_dir): @@ -3795,7 +3795,7 @@ def make_module_step(self, fake=False): try: self.make_devel_module() except EasyBuildError as error: - if build_option('module_only'): + if build_option('module_only') or self.cfg['module_only']: self.log.info("Using --module-only so can recover from error: %s", error) else: raise error @@ -3903,7 +3903,7 @@ def skip_step(self, step, skippable): """Dedice whether or not to skip the specified step.""" skip = False force = build_option('force') - module_only = build_option('module_only') + module_only = build_option('module_only') or self.cfg['module_only'] sanity_check_only = build_option('sanity_check_only') skip_extensions = build_option('skip_extensions') skip_test_step = build_option('skip_test_step') diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 965e6db037..ce11d0457b 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -109,6 +109,7 @@ 'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD], 'installopts': ['', 'Extra options for installation', BUILD], 'maxparallel': [None, 'Max degree of parallelism', BUILD], + 'module_only': [False, 'Only generate module file', BUILD], 'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of ' 'cores, active cpuset and restrictions in ulimit)'), BUILD], 'patches': [[], "List of patches to apply", BUILD], From 27526c7dfad2d1cad2d4e42fc6af43b970f9aa15 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 22 May 2024 09:21:07 +0200 Subject: [PATCH 311/430] move CWD_NOTFOUND_ERROR constant to tools.build_log --- easybuild/tools/build_log.py | 5 +++++ easybuild/tools/filetools.py | 4 ++-- easybuild/tools/run.py | 7 +------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 2e2bf726c0..4880415151 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -61,6 +61,11 @@ DRY_RUN_SOFTWARE_INSTALL_DIR = None DRY_RUN_MODULES_INSTALL_DIR = None +CWD_NOTFOUND_ERROR = ( + "Current working directory does not exist! It was either unexpectedly removed " + "by an external process to EasyBuild or the filesystem is misbehaving." +) + DEVEL_LOG_LEVEL = logging.DEBUG - 1 logging.addLevelName(DEVEL_LOG_LEVEL, 'DEVEL') diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 133840efc2..e20b876196 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -62,11 +62,11 @@ from easybuild.base import fancylogger # import build_log must stay, to use of EasyBuildLog -from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning +from easybuild.tools.build_log import EasyBuildError, CWD_NOTFOUND_ERROR, dry_run_msg, print_msg, print_warning from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.hooks import load_source -from easybuild.tools.run import CWD_NOTFOUND_ERROR, run_shell_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index b8196940f4..b8d3eb310e 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -63,7 +63,7 @@ from threading import get_ident as get_thread_id from easybuild.base import fancylogger -from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since +from easybuild.tools.build_log import EasyBuildError, CWD_NOTFOUND_ERROR, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import build_option from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.utilities import trace_msg @@ -83,11 +83,6 @@ "ulimit -u", # used in det_parallelism ) -CWD_NOTFOUND_ERROR = ( - "Current working directory does not exist! It was either unexpectedly removed " - "by an external process to EasyBuild or the filesystem is misbehaving." -) - RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', 'out_file', 'err_file', 'thread_id', 'task_id')) From 7749bec88342a439263be1ba8e0afef3e2f97af0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 22 May 2024 18:16:12 +0200 Subject: [PATCH 312/430] restore original values of easyconfig.templates.DEPRECATED_TEMPLATES & co by simply reloading easyconfig.templates and easyconfig.easyconfig modules --- test/framework/easyconfig.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index b29c675a78..e24cd72604 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -120,11 +120,6 @@ def setUp(self): github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None - self.orig_alternate_constants = copy.deepcopy(easyconfig.templates.ALTERNATE_TEMPLATE_CONSTANTS) - self.orig_alternate_templates = copy.deepcopy(easyconfig.templates.ALTERNATE_TEMPLATES) - self.orig_deprecated_constants = copy.deepcopy(easyconfig.templates.DEPRECATED_TEMPLATE_CONSTANTS) - self.orig_deprecated_templates = copy.deepcopy(easyconfig.templates.DEPRECATED_TEMPLATES) - def prep(self): """Prepare for test.""" # (re)cleanup last test file @@ -139,11 +134,10 @@ def tearDown(self): """ make sure to remove the temporary file """ st.get_cpu_architecture = self.orig_get_cpu_architecture - easyconfig.templates.ALTERNATE_TEMPLATE_CONSTANTS = self.orig_alternate_constants - easyconfig.templates.ALTERNATE_TEMPLATES = self.orig_alternate_templates - easyconfig.templates.DEPRECATED_TEMPLATE_CONSTANTS = self.orig_deprecated_constants - easyconfig.templates.DEPRECATED_TEMPLATES = self.orig_deprecated_templates + # reload easyconfig.template module to restore orignal values of DEPRECATED_TEMPLATES & co reload(easyconfig.templates) + # also reload easyconfig.easyconfig module, which imports from easyconfig.templates + reload(easyconfig.easyconfig) super(EasyConfigTest, self).tearDown() if os.path.exists(self.eb_file): From d62315426e2406af3c4ce8640e134f4f36220e15 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 22 May 2024 19:35:05 +0200 Subject: [PATCH 313/430] restore DEPRECATED_TEMPLATES & co in easyconfig.templates without reloading, since that messes up isinstance checks using EasyConfig class --- test/framework/easyconfig.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 09dcd8f6de..9123ada14e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -120,6 +120,9 @@ def setUp(self): github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None + self.orig_easyconfig_DEPRECATED_TEMPLATES = easyconfig.easyconfig.DEPRECATED_TEMPLATES + self.orig_easyconfig_ALTERNATE_TEMPLATES = easyconfig.easyconfig.ALTERNATE_TEMPLATES + def prep(self): """Prepare for test.""" # (re)cleanup last test file @@ -134,15 +137,14 @@ def tearDown(self): """ make sure to remove the temporary file """ st.get_cpu_architecture = self.orig_get_cpu_architecture - # reload easyconfig.template module to restore orignal values of DEPRECATED_TEMPLATES & co - reload(easyconfig.templates) - # also reload easyconfig.easyconfig module, which imports from easyconfig.templates - reload(easyconfig.easyconfig) - super(EasyConfigTest, self).tearDown() if os.path.exists(self.eb_file): os.remove(self.eb_file) + # restore orignal values of DEPRECATED_TEMPLATES & co in easyconfig.templates + easyconfig.easyconfig.DEPRECATED_TEMPLATES = self.orig_easyconfig_DEPRECATED_TEMPLATES + easyconfig.easyconfig.ALTERNATE_TEMPLATES = self.orig_easyconfig_ALTERNATE_TEMPLATES + def test_empty(self): """ empty files should not parse! """ self.contents = "# empty string" @@ -1467,13 +1469,13 @@ def test_template_deprecation_and_alternate(self): 'cudaver': ('depr_cuda_ver', '1000000000'), 'start_dir': ('depr_start_dir', '1000000000'), } - easyconfig.templates.DEPRECATED_TEMPLATES.update(template_test_deprecations) + easyconfig.easyconfig.DEPRECATED_TEMPLATES = template_test_deprecations template_test_alternates = { 'installdir': 'alt_install_dir', 'version_major_minor': 'alt_ver_maj_min', } - easyconfig.templates.ALTERNATE_TEMPLATES.update(template_test_alternates) + easyconfig.easyconfig.ALTERNATE_TEMPLATES = template_test_alternates tmpl_str = ("cd %(start_dir)s && make %(namelower)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" " && echo %(alt_install_dir)s %(version_major_minor)s") From 64cde9e811ceca718fef21d58c11163abdfc81dd Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Thu, 23 May 2024 11:28:48 +0200 Subject: [PATCH 314/430] add test --- test/framework/options.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 3deaf8cf9c..38f4eea2e7 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -366,6 +366,16 @@ def test_skip(self): self.assertEqual(len(glob.glob(toy_mod_glob)), 1) + for toy_mod in glob.glob(toy_mod_glob): + remove_file(toy_mod) + + # check use of module_only parameter + test_ec_txt += "\nmodule_only = True\n" + write_file(test_ec, test_ec_txt) + self.eb_main(args, do_build=True, raise_error=True) + + self.assertEqual(len(glob.glob(toy_mod_glob)), 1) + def test_skip_test_step(self): """Test skipping testing the build (--skip-test-step).""" From 8a5ac07a2013ae8c8ffae08617030c9bc14ed27c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 May 2024 08:46:40 +0200 Subject: [PATCH 315/430] add test_alternate_easyconfig_parameters + clean up/fix test_deprecated_easyconfig_parameters to prevent cross-test leaking --- test/framework/easyconfig.py | 64 ++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 9123ada14e..d292fdbce0 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -120,7 +120,9 @@ def setUp(self): github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None + self.orig_easyconfig_DEPRECATED_PARAMETERS = easyconfig.easyconfig.DEPRECATED_PARAMETERS self.orig_easyconfig_DEPRECATED_TEMPLATES = easyconfig.easyconfig.DEPRECATED_TEMPLATES + self.orig_easyconfig_ALTERNATE_PARAMETERS = easyconfig.easyconfig.ALTERNATE_PARAMETERS self.orig_easyconfig_ALTERNATE_TEMPLATES = easyconfig.easyconfig.ALTERNATE_TEMPLATES def prep(self): @@ -142,7 +144,9 @@ def tearDown(self): os.remove(self.eb_file) # restore orignal values of DEPRECATED_TEMPLATES & co in easyconfig.templates + easyconfig.easyconfig.DEPRECATED_PARAMETERS = self.orig_easyconfig_DEPRECATED_PARAMETERS easyconfig.easyconfig.DEPRECATED_TEMPLATES = self.orig_easyconfig_DEPRECATED_TEMPLATES + easyconfig.easyconfig.ALTERNATE_PARAMETERS = self.orig_easyconfig_ALTERNATE_PARAMETERS easyconfig.easyconfig.ALTERNATE_TEMPLATES = self.orig_easyconfig_ALTERNATE_TEMPLATES def test_empty(self): @@ -1815,6 +1819,35 @@ def foo(key): self.assertErrorRegex(EasyBuildError, error_regex, foo, key) + def test_alternate_easyconfig_parameters(self): + """Test handling of alternate easyconfig parameters.""" + + test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec).replace('postinstallcmds', 'post_install_cmds').replace('moduleclass', 'env_mod_class') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + # post_install_cmds is not accepted unless it's registered as an alternative easyconfig parameter + easyconfig.easyconfig.ALTERNATE_PARAMETERS = {} + self.assertErrorRegex(EasyBuildError, "post_install_cmds -> postinstallcmds", EasyConfig, test_ec) + + easyconfig.easyconfig.ALTERNATE_PARAMETERS = { + 'env_mod_class': 'moduleclass', + 'post_install_cmds': 'postinstallcmds', + } + ec = EasyConfig(test_ec) + + expected = 'tools' + self.assertEqual(ec['moduleclass'], expected) + self.assertEqual(ec['env_mod_class'], expected) + + expected = ['echo TOY > %(installdir)s/README'] + self.assertEqual(ec['postinstallcmds'], expected) + self.assertEqual(ec['post_install_cmds'], expected) + def test_deprecated_easyconfig_parameters(self): """Test handling of deprecated easyconfig parameters.""" os.environ.pop('EASYBUILD_DEPRECATED') @@ -1824,21 +1857,15 @@ def test_deprecated_easyconfig_parameters(self): test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) - orig_deprecated_parameters = copy.deepcopy(easyconfig.parser.DEPRECATED_PARAMETERS) - easyconfig.parser.DEPRECATED_PARAMETERS.update({ + easyconfig.easyconfig.DEPRECATED_PARAMETERS = { 'foobar': ('barfoo', '0.0'), # deprecated since forever # won't be actually deprecated for a while; # note that we should map foobarbarfoo to a valid easyconfig parameter here, # or we'll hit errors when parsing an easyconfig file that uses it 'foobarbarfoo': ('required_linked_shared_libs', '1000000000'), - }) - - # copy classes before reloading, so we can restore them (otherwise isinstance checks fail) - orig_EasyConfig = copy.deepcopy(easyconfig.easyconfig.EasyConfig) - orig_ActiveMNS = copy.deepcopy(easyconfig.easyconfig.ActiveMNS) - reload(easyconfig.parser) + } - for key, (newkey, depr_ver) in easyconfig.parser.DEPRECATED_PARAMETERS.items(): + for key, (newkey, depr_ver) in easyconfig.easyconfig.DEPRECATED_PARAMETERS.items(): if LooseVersion(depr_ver) <= easybuild.tools.build_log.CURRENT_VERSION: # deprecation error error_regex = "DEPRECATED.*since v%s.*'%s' is deprecated.*use '%s' instead" % (depr_ver, key, newkey) @@ -1851,12 +1878,13 @@ def foo(key): self.assertErrorRegex(EasyBuildError, error_regex, foo, key) else: # only deprecation warning, but key is replaced when getting/setting - ec[key] = 'test123' - self.assertEqual(ec[newkey], 'test123') - self.assertEqual(ec[key], 'test123') - ec[newkey] = '123test' - self.assertEqual(ec[newkey], '123test') - self.assertEqual(ec[key], '123test') + with self.mocked_stdout_stderr(): + ec[key] = 'test123' + self.assertEqual(ec[newkey], 'test123') + self.assertEqual(ec[key], 'test123') + ec[newkey] = '123test' + self.assertEqual(ec[newkey], '123test') + self.assertEqual(ec[key], '123test') variables = { 'name': 'example', @@ -1887,12 +1915,6 @@ def foo(key): ec = EasyConfig(test_ec) self.assertEqual(ec['required_linked_shared_libs'], 'foobarbarfoo') - easyconfig.parser.DEPRECATED_PARAMETERS = orig_deprecated_parameters - reload(easyconfig.parser) - reload(easyconfig.easyconfig) - easyconfig.easyconfig.EasyConfig = orig_EasyConfig - easyconfig.easyconfig.ActiveMNS = orig_ActiveMNS - def test_unknown_easyconfig_parameter(self): """Check behaviour when unknown easyconfig parameters are used.""" self.contents = '\n'.join([ From 8e52d81fe43dbdf6ce2ad89366c3ff4140278770 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 May 2024 09:04:19 +0200 Subject: [PATCH 316/430] fix long line in test_alternate_easyconfig_parameters --- test/framework/easyconfig.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index d292fdbce0..b5c5183fab 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1825,7 +1825,9 @@ def test_alternate_easyconfig_parameters(self): test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - test_ec_txt = read_file(toy_ec).replace('postinstallcmds', 'post_install_cmds').replace('moduleclass', 'env_mod_class') + test_ec_txt = read_file(toy_ec) + test_ec_txt = test_ec_txt.replace('postinstallcmds', 'post_install_cmds') + test_ec_txt = test_ec_txt.replace('moduleclass', 'env_mod_class') test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) From a86f26b32f834fe72a7b0ba6efe629179913e548 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 May 2024 22:53:08 +0200 Subject: [PATCH 317/430] extend test_alternate_easyconfig_parameters to also check setting/updating of easyconfig parameter values --- test/framework/easyconfig.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index b5c5183fab..1807ec8d0c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -40,7 +40,6 @@ import textwrap from collections import OrderedDict from easybuild.tools import LooseVersion -from importlib import reload from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -1850,6 +1849,21 @@ def test_alternate_easyconfig_parameters(self): self.assertEqual(ec['postinstallcmds'], expected) self.assertEqual(ec['post_install_cmds'], expected) + # test setting of easyconfig parameter with original & alternate name + ec['moduleclass'] = 'test1' + self.assertEqual(ec['moduleclass'], 'test1') + self.assertEqual(ec['env_mod_class'], 'test1') + ec.update('moduleclass', 'test2') + self.assertEqual(ec['moduleclass'], 'test1 test2 ') + self.assertEqual(ec['env_mod_class'], 'test1 test2 ') + + ec['env_mod_class'] = 'test3' + self.assertEqual(ec['moduleclass'], 'test3') + self.assertEqual(ec['env_mod_class'], 'test3') + ec.update('env_mod_class', 'test4') + self.assertEqual(ec['moduleclass'], 'test3 test4 ') + self.assertEqual(ec['env_mod_class'], 'test3 test4 ') + def test_deprecated_easyconfig_parameters(self): """Test handling of deprecated easyconfig parameters.""" os.environ.pop('EASYBUILD_DEPRECATED') From 430189c2dfd39b5ea494b66b06ab6acfa347851d Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 28 May 2024 23:10:25 +0200 Subject: [PATCH 318/430] redifine args and test that no software was installed --- test/framework/options.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 38f4eea2e7..ba5e6602f4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -366,16 +366,25 @@ def test_skip(self): self.assertEqual(len(glob.glob(toy_mod_glob)), 1) - for toy_mod in glob.glob(toy_mod_glob): - remove_file(toy_mod) - # check use of module_only parameter + remove_dir(os.path.join(self.test_installpath, 'modules', 'all', 'toy')) + remove_dir(os.path.join(self.test_installpath, 'software', 'toy', '0.0')) + args = [ + test_ec, + '--rebuild', + ] test_ec_txt += "\nmodule_only = True\n" write_file(test_ec, test_ec_txt) self.eb_main(args, do_build=True, raise_error=True) self.assertEqual(len(glob.glob(toy_mod_glob)), 1) + # check that no software was installed + installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + installdir_glob = glob.glob(os.path.join(installdir, '*')) + easybuild_dir = os.path.join(installdir, 'easybuild') + self.assertEqual(installdir_glob, [easybuild_dir]) + def test_skip_test_step(self): """Test skipping testing the build (--skip-test-step).""" From d215764729407f36695c2b7e434c1ecdb665da85 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 23 Apr 2024 15:24:48 +0100 Subject: [PATCH 319/430] add renamed easyconfig params to ALTERNATE_PARAMTERS --- easybuild/framework/easyconfig/parser.py | 53 ++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index 5cab3b3ddf..6860eb87fd 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -45,6 +45,59 @@ # alternate easyconfig parameters, and their non-deprecated equivalents ALTERNATE_PARAMETERS = { # : , + 'build_deps': 'builddependencies', + 'build_in_install_dir': 'buildininstalldir', + 'build_opts': 'buildopts', + 'build_stats': 'buildstats', + 'clean_up_old_build': 'cleanupoldbuild', + 'clean_up_old_install': 'cleanupoldinstall', + 'configure_opts': 'configopts', + 'deps': 'dependencies', + 'doc_paths': 'docpaths', + 'doc_urls': 'docurls', + 'dont_create_install_dir': 'dontcreateinstalldir', + 'exts_class_map': 'exts_classmap', + 'exts_default_class': 'exts_default_class', + 'exts_default_opts': 'exts_default_options', + 'hidden_deps': 'hiddendependencies', + 'include_modulepath_exts': 'include_modpath_extensions', + 'install_opts': 'installopts', + 'keep_previous_install': 'keeppreviousinstall', + 'keep_symlinks': 'keepsymlinks', + 'max_parallel': 'maxparallel', + 'env_mod_aliases': 'modaliases', + 'env_mod_alt_soft_name': 'modaltsoftname', + 'modulepath_prepend_paths': 'moddependpaths', + 'env_mod_extra_paths_append': 'modextrapaths_append', + 'env_mod_extra_paths': 'modextrapaths', + 'env_mod_extra_vars': 'modextravars', + 'env_mod_load_msg': 'modloadmsg', + 'env_mod_lua_footer': 'modluafooter', + 'env_mod_tcl_footer': 'modtclfooter', + 'env_mod_class': 'moduleclass', + 'env_mod_depends_on': 'module_depends_on', + 'env_mod_force_unload': 'moduleforceunload', + 'env_mod_load_no_conflict': 'moduleloadnoconflict', + 'env_mod_unload_msg': 'modunloadmsg', + 'only_toolchain_mod_env': 'onlytcmod', + 'os_deps': 'osdependencies', + 'post_install_cmds': 'postinstallcmds', + 'post_install_msgs': 'postinstallmsgs', + 'post_install_patches': 'postinstallpatches', + 'pre_build_opts': 'prebuildopts', + 'pre_configure_opts': 'preconfigopts', + 'pre_install_opts': 'preinstallopts', + 'pre_test_opts': 'pretestopts', + 'recursive_env_mod_unload': 'recursive_module_unload', + 'run_test': 'runtest', + 'sanity_check_cmds': 'sanity_check_commands', + 'skip_fortran_mod_files_sanity_check': 'skip_mod_files_sanity_check', + 'skip_steps': 'skipsteps', + 'test_opts': 'testopts', + 'toolchain_opts': 'toolchainopts', + 'unpack_opts': 'unpack_options', + 'version_prefix': 'versionprefix', + 'version_suffix': 'versionsuffix', } # deprecated easyconfig parameters, and their replacements From 9dc92397885711a20bb0874d1c226ff2b8c726b6 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 23 Apr 2024 15:26:10 +0100 Subject: [PATCH 320/430] add renamed templates to ALTERNATE_TEMPLATES --- easybuild/framework/easyconfig/templates.py | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 7ee17d2366..4ec9b4a25a 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -164,6 +164,35 @@ # alternate templates, and their equivalents ALTERNATE_TEMPLATES = { # : , + 'build_dir': 'builddir', + 'cuda_maj_ver': 'cudamajver', + 'cuda_maj_ver': 'cudamajver', + 'cuda_short_ver': 'cudashortver', + 'cuda_short_ver': 'cudashortver', + 'cuda_ver': 'cudaver', + 'cuda_ver': 'cudaver', + 'install_dir': 'installdir', + 'java_maj_ver': 'javamajver', + 'java_short_ver': 'javashortver', + 'java_ver': 'javaver', + 'name_letter_lower': 'nameletterlower', + 'name_letter': 'nameletter', + 'name_lower': 'namelower', + 'perl_maj_ver': 'perlmajver', + 'perl_short_ver': 'perlshortver', + 'perl_ver': 'perlver', + 'py_maj_ver': 'pymajver', + 'py_short_ver': 'pyshortver', + 'py_ver': 'pyver', + 'r_maj_ver': 'rmajver', + 'r_short_ver': 'rshortver', + 'r_ver': 'rver', + 'toolchain_ver': 'toolchain_version', + 'ver_maj_min': 'version_major_minor', + 'ver_maj': 'version_major', + 'ver_min': 'version_minor', + 'version_prefix': 'versionprefix', + 'version_suffix': 'versionsuffix', } # deprecated templates, and their replacements From 707d8b24e429fb81a89c69f2f5e698a193415d5d Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 23 Apr 2024 15:33:23 +0100 Subject: [PATCH 321/430] add renamed template constants to ALTERNATE_TEMPLATE_CONSTANTS --- easybuild/framework/easyconfig/templates.py | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 4ec9b4a25a..139978396a 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -203,6 +203,43 @@ # alternate template constants, and their equivalents ALTERNATE_TEMPLATE_CONSTANTS = { # : , + 'APACHE_URL': 'APACHE_SOURCE', + 'BITBUCKET_GET_URL': 'BITBUCKET_SOURCE', + 'BITBUCKET_DOWNLOADS_URL': 'BITBUCKET_DOWNLOADS', + 'CRAN_URL': 'CRAN_SOURCE', + 'FTP_GNOME_URL': 'FTPGNOME_SOURCE', + 'GITHUB_URL': 'GITHUB_SOURCE', + 'GITHUB_URL_LOWER': 'GITHUB_LOWER_SOURCE', + 'GITHUB_RELEASE_URL': 'GITHUB_RELEASE', + 'GITHUB_RELEASE_URL_LOWER': 'GITHUB_LOWER_RELEASE', + 'GNU_SAVANNAH_URL': 'GNU_SAVANNAH_SOURCE', + 'GNU_URL': 'GNU_SOURCE', + 'GOOGLECODE_URL': 'GOOGLECODE_SOURCE', + 'LAUNCHPAD_URL': 'LAUNCHPAD_SOURCE', + 'PYPI_URL': 'PYPI_SOURCE', + 'PYPI_URL_LOWER': 'PYPI_LOWER_SOURCE', + 'R_URL': 'R_SOURCE', + 'SOURCEFORGE_URL': 'SOURCEFORGE_SOURCE', + 'XORG_DATA_URL': 'XORG_DATA_SOURCE', + 'XORG_LIB_URL': 'XORG_LIB_SOURCE', + 'XORG_PROTO_URL': 'XORG_PROTO_SOURCE', + 'XORG_UTIL_URL': 'XORG_UTIL_SOURCE', + 'XORG_XCB_URL': 'XORG_XCB_SOURCE', + 'SOURCE_LOWER_TAR_GZ': 'SOURCELOWER_TAR_GZ', + 'SOURCE_LOWER_TAR_XZ': 'SOURCELOWER_TAR_XZ', + 'SOURCE_LOWER_TAR_BZ2': 'SOURCELOWER_TAR_BZ2', + 'SOURCE_LOWER_TGZ': 'SOURCELOWER_TGZ', + 'SOURCE_LOWER_TXZ': 'SOURCELOWER_TXZ', + 'SOURCE_LOWER_TBZ2': 'SOURCELOWER_TBZ2', + 'SOURCE_LOWER_TB2': 'SOURCELOWER_TB2', + 'SOURCE_LOWER_GTGZ': 'SOURCELOWER_GTGZ', + 'SOURCE_LOWER_ZIP': 'SOURCELOWER_ZIP', + 'SOURCE_LOWER_TAR': 'SOURCELOWER_TAR', + 'SOURCE_LOWER_XZ': 'SOURCELOWER_XZ', + 'SOURCE_LOWER_TAR_Z': 'SOURCELOWER_TAR_Z', + 'SOURCE_LOWER_WHL': 'SOURCELOWER_WHL', + 'SOURCE_LOWER_PY2_WHL': 'SOURCELOWER_PY2_WHL', + 'SOURCE_LOWER_PY3_WHL': 'SOURCELOWER_PY3_WHL', } # deprecated template constants, and their replacements From f5122c7d5ae411cc533159f17fbf55a8f39414ee Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Wed, 29 May 2024 13:24:24 +0100 Subject: [PATCH 322/430] add alternate template for cuda_compute_capabilities -> cuda_cc_comma_sep --- easybuild/framework/easyconfig/templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 139978396a..737b105915 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -165,6 +165,7 @@ ALTERNATE_TEMPLATES = { # : , 'build_dir': 'builddir', + 'cuda_cc_comma_sep': 'cuda_compute_capabilities', 'cuda_maj_ver': 'cudamajver', 'cuda_maj_ver': 'cudamajver', 'cuda_short_ver': 'cudashortver', From 62b9e9d96eaa5c8583db2cf592679a44463cfdcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Thu, 30 May 2024 11:38:43 +0200 Subject: [PATCH 323/430] Move run_shell_cmd hook before cmd_str is defined. --- easybuild/tools/run.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index b8d3eb310e..14335d5f45 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -319,6 +319,17 @@ def to_cmd_str(cmd): except FileNotFoundError: raise EasyBuildError(CWD_NOTFOUND_ERROR) + if with_hooks: + hooks = load_hooks(build_option('hooks')) + kwargs = { + 'interactive': interactive, + 'work_dir': work_dir, + } + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=kwargs) + if hook_res: + cmd, old_cmd = hook_res, cmd + _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) + cmd_str = to_cmd_str(cmd) thread_id = None @@ -378,18 +389,6 @@ def to_cmd_str(cmd): else: executable, shell = None, False - if with_hooks: - hooks = load_hooks(build_option('hooks')) - kwargs = { - 'interactive': interactive, - 'work_dir': work_dir, - } - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=kwargs) - if hook_res: - cmd, old_cmd = hook_res, cmd - cmd_str = to_cmd_str(cmd) - _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) - stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT log_msg = f"Running {interactive_msg}shell command '{cmd_str}' in {work_dir}" From 41a3ce768f88f437c415332f9dbbff56f0a79346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Thu, 30 May 2024 11:43:04 +0200 Subject: [PATCH 324/430] Fix defining "interactive" flag before hook --- easybuild/tools/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 14335d5f45..79fabb7e47 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -310,6 +310,8 @@ def to_cmd_str(cmd): if not isinstance(qa_patterns, list) or any(not isinstance(x, tuple) or len(x) != 2 for x in qa_patterns): raise EasyBuildError("qa_patterns passed to run_shell_cmd should be a list of 2-tuples!") + interactive = bool(qa_patterns) + if qa_wait_patterns is None: qa_wait_patterns = [] @@ -358,7 +360,6 @@ def to_cmd_str(cmd): else: cmd_out_fp, cmd_err_fp = None, None - interactive = bool(qa_patterns) interactive_msg = 'interactive ' if interactive else '' # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled) From 4ddad9bbe1dfae50e5c721212c6ead07911f7135 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 30 May 2024 12:14:29 +0200 Subject: [PATCH 325/430] minor tweaks to dumping of env.sh + run.sh helper scripts in run_shell_cmd + enhance test to verify they're working as intended --- easybuild/tools/run.py | 33 ++++++++++++++++++++++----------- test/framework/run.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 2069979ae4..18fdad4e89 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -204,30 +204,38 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): return ''.join([c for c in cmd if c in allowed_chars]) -def save_cmd(cmd_str, work_dir, env, tmpdir): - # Save environment variables in it's own environment file +def create_cmd_scripts(cmd_str, work_dir, env, tmpdir): + """ + Create helper scripts for specified command in specified directory: + - env.sh which can be sourced to define environment in which command was run; + - cmd.sh to create interactive (bash) shell session with working directory and environment, + and with the command in shell history; + """ + # Save environment variables in env.sh which can be sourced to restore environment full_env = os.environ.copy() if env is not None: full_env.update(env) env_fp = os.path.join(tmpdir, 'env.sh') with open(env_fp, 'w') as fid: # excludes bash functions (environment variables ending with %) - fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in full_env.items() + fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in sorted(full_env.items()) if not key.endswith('%'))) fid.write('\n\nPS1="eb-shell> "') + # also change to working directory (to ensure that working directory is correct for interactive bash shell) + fid.write(f'\ncd "{work_dir}"') fid.write(f'\nhistory -s {shlex.quote(cmd_str)}') - # Make script that sets up bash shell with given environments set. + # Make script that sets up bash shell with specified environment and working directory cmd_fp = os.path.join(tmpdir, 'cmd.sh') with open(cmd_fp, 'w') as fid: fid.write('#!/usr/bin/env bash\n') - fid.write('# Run this script to replicate the environment that EB used to run the shell command\n') + fid.write('# Run this script to set up a shell environment that EasyBuild used to run the shell command\n') fid.write('\n'.join([ - f'\ncd "{work_dir}"', 'EB_SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )', - f'echo Shell for the command: {shlex.quote(cmd_str)}', - 'echo Use command history, exit to stop', - 'bash --rcfile $EB_SCRIPT_DIR/env.sh', + f'echo "# Shell for the command: {shlex.quote(cmd_str)}"', + 'echo "# Use command history, exit to stop"', + # using -i to force interactive shell, so env.sh is also sourced when -c is used to run commands + 'bash --rcfile $EB_SCRIPT_DIR/env.sh -i "$@"', ])) os.chmod(cmd_fp, 0o775) @@ -356,14 +364,17 @@ def to_cmd_str(cmd): _log.info(f"Auto-enabling streaming output of '{cmd_str}' command because logging to stdout is enabled") stream_output = True - # temporary output file(s) for command output + # temporary output file(s) for command output, along with helper scripts if output_file: toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') os.makedirs(toptmpdir, exist_ok=True) cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') + _log.info(f'run_shell_cmd: command environment of "{cmd_str}" will be saved to {tmpdir}') - save_cmd(cmd_str, work_dir, env, tmpdir) + + create_cmd_scripts(cmd_str, work_dir, env, tmpdir) + cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_shell_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') if split_stderr: diff --git a/test/framework/run.py b/test/framework/run.py index f1ee6955ec..88546ca136 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -167,6 +167,10 @@ def test_run_cmd(self): def test_run_shell_cmd_basic(self): """Basic test for run_shell_cmd function.""" + os.environ['FOOBAR'] = 'foobar' + + cwd = change_dir(self.test_prefix) + with self.mocked_stdout_stderr(): res = run_shell_cmd("echo hello") self.assertEqual(res.output, "hello\n") @@ -177,6 +181,42 @@ def test_run_shell_cmd_basic(self): self.assertEqual(res.stderr, None) self.assertTrue(res.work_dir and isinstance(res.work_dir, str)) + change_dir(cwd) + del os.environ['FOOBAR'] + + # check on helper scripts that were generated for this command + paths = glob.glob(os.path.join(self.test_prefix, 'eb-*', 'run-shell-cmd-output', 'echo-*')) + self.assertEqual(len(paths), 1) + cmd_tmpdir = paths[0] + + # check on env.sh script that can be used to set up environment in which command was run + env_script = os.path.join(cmd_tmpdir, 'env.sh') + self.assertExists(env_script) + env_script_txt = read_file(env_script) + self.assertIn("export FOOBAR=foobar", env_script_txt) + self.assertIn("history -s 'echo hello'", env_script_txt) + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(f"source {env_script}; echo $FOOBAR; history") + self.assertEqual(res.exit_code, 0) + self.assertTrue(res.output.startswith('foobar\n')) + self.assertTrue(res.output.endswith("echo hello\n")) + + # check on cmd.sh script that can be used to create interactive shell environment for command + cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh') + self.assertExists(cmd_script) + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(f"{cmd_script} -c 'echo pwd: $PWD; echo $FOOBAR'", fail_on_error=False) + self.assertEqual(res.exit_code, 0) + self.assertTrue(res.output.endswith('foobar\n')) + # check whether working directory is what's expected + regex = re.compile('^pwd: .*', re.M) + res = regex.findall(res.output) + self.assertEqual(len(res), 1) + pwd = res[0].strip()[5:] + self.assertTrue(os.path.samefile(pwd, self.test_prefix)) + # test running command that emits non-UTF-8 characters # this is constructed to reproduce errors like: # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 From 1b2d8638470eb2d7b5d3d62cc5ff14f8573f9287 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 30 May 2024 14:52:25 +0200 Subject: [PATCH 326/430] fix test_update_branch_github which got broken because toy-0.0.eb easyconfig was modified --- test/framework/options.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 77764440dd..00c8af314a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4485,7 +4485,7 @@ def test_new_branch_github(self): def test_github_new_pr_from_branch(self): """Test --new-pr-from-branch.""" if self.github_token is None: - print("Skipping test_new_pr_from_branch, no GitHub token available?") + print("Skipping test_github_new_pr_from_branch, no GitHub token available?") return # see https://github.com/boegel/easybuild-easyconfigs/tree/test_new_pr_from_branch_DO_NOT_REMOVE @@ -4517,8 +4517,8 @@ def test_github_new_pr_from_branch(self): r"^\* from: boegel/easybuild-easyconfigs:test_new_pr_from_branch_DO_NOT_REMOVE$", r'^\* title: "\{tools\}\[system/system\] toy v0\.0"$', r'^"an easyconfig for toy"$', - r"^ 1 file changed, 32 insertions\(\+\)$", - r"^\* overview of changes:\n easybuild/easyconfigs/t/toy/toy-0\.0\.eb | 32", + r"^ 1 file changed, [0-9]+ insertions\(\+\)$", + r"^\* overview of changes:\n easybuild/easyconfigs/t/toy/toy-0\.0\.eb | [0-9]+", ] self._assert_regexs(regexs, txt) @@ -4546,7 +4546,7 @@ def test_update_branch_github(self): r"^== fetching branch 'develop' from https://github.com/%s.git\.\.\." % full_repo, r"^== copying files to .*/git-working-dir.*/easybuild-easyconfigs...", r"^== pushing branch 'develop' to remote '.*' \(git@github.com:%s.git\) \[DRY RUN\]" % full_repo, - r"^Overview of changes:\n.*/easyconfigs/t/toy/toy-0.0.eb \| 32", + r"^Overview of changes:\n.*/easyconfigs/t/toy/toy-0.0.eb \| [0-9]+", r"== pushed updated branch 'develop' to boegel/easybuild-easyconfigs \[DRY RUN\]", ] self._assert_regexs(regexs, txt) From 3338c610ac3cde1309a1580b18b24041009ecda8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 30 May 2024 21:15:31 +0200 Subject: [PATCH 327/430] fix handling of specified command environment in create_cmd_scripts + add test for env option in run_shell_cmd --- easybuild/tools/run.py | 17 ++++++++++++----- test/framework/run.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index e3fd252853..7b313c894b 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -205,17 +205,24 @@ def create_cmd_scripts(cmd_str, work_dir, env, tmpdir): and with the command in shell history; """ # Save environment variables in env.sh which can be sourced to restore environment - full_env = os.environ.copy() - if env is not None: - full_env.update(env) + if env is None: + env = os.environ.copy() + env_fp = os.path.join(tmpdir, 'env.sh') with open(env_fp, 'w') as fid: # excludes bash functions (environment variables ending with %) - fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in sorted(full_env.items()) - if not key.endswith('%'))) + fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in sorted(env.items()) + if not key.endswith('%')) + '\n') + + # unset environment variables in current environment if they're not defined in environment used to run command + fid.write('\n'.join(f'unset {key}' for key in os.environ if key not in env and not key.endswith('%'))) + fid.write('\n\nPS1="eb-shell> "') + # also change to working directory (to ensure that working directory is correct for interactive bash shell) fid.write(f'\ncd "{work_dir}"') + + # reset shell history to only include executed command fid.write(f'\nhistory -s {shlex.quote(cmd_str)}') # Make script that sets up bash shell with specified environment and working directory diff --git a/test/framework/run.py b/test/framework/run.py index d9eab214e2..bc32a877a9 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -247,6 +247,40 @@ def test_run_shell_cmd_basic(self): self.assertTrue(isinstance(res.output, str)) self.assertTrue(res.work_dir and isinstance(res.work_dir, str)) + def test_run_shell_cmd_env(self): + """Test env option in run_shell_cmd.""" + + # use 'env' to define environment in which command should be run; + # with a few exceptions (like $_, $PWD) no other environment variables will be defined, + # so $HOME and $USER will not be set + cmd = "env | sort" + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, env={'FOOBAR': 'foobar', 'PATH': os.getenv('PATH')}) + self.assertEqual(res.cmd, cmd) + self.assertEqual(res.exit_code, 0) + self.assertIn("FOOBAR=foobar\n", res.output) + self.assertTrue(re.search("^_=.*/env$", res.output, re.M)) + for var in ('HOME', 'USER'): + self.assertFalse(re.search('^' + var + '=.*', res.output, re.M)) + + # check on helper scripts that were generated for this command + paths = glob.glob(os.path.join(self.test_prefix, 'eb-*', 'run-shell-cmd-output', 'env-*')) + self.assertEqual(len(paths), 1) + cmd_tmpdir = paths[0] + + env_script = os.path.join(cmd_tmpdir, 'env.sh') + self.assertExists(env_script) + env_script_txt = read_file(env_script) + self.assertTrue(env_script_txt.startswith('export FOOBAR=foobar\nexport PATH')) + + cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh') + self.assertExists(cmd_script) + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(f"{cmd_script} -c 'echo $FOOBAR'", fail_on_error=False) + self.assertEqual(res.exit_code, 0) + self.assertTrue(res.output.endswith('\nfoobar\n')) + def test_fileprefix_from_cmd(self): """test simplifications from fileprefix_from_cmd.""" cmds = { From a2b9e06014c11db2d4b9ae2d35ed30125f3fb189 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 31 May 2024 09:13:37 +0200 Subject: [PATCH 328/430] fix unsetting of current environment in env.sh script produced by run_shell_cmd --- easybuild/tools/run.py | 8 +++++--- test/framework/run.py | 16 +++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 7b313c894b..737d7dad9e 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -210,13 +210,15 @@ def create_cmd_scripts(cmd_str, work_dir, env, tmpdir): env_fp = os.path.join(tmpdir, 'env.sh') with open(env_fp, 'w') as fid: + # unset all environment variables in current environment first to start from a clean slate; + # we need to be careful to filter out functions definitions, so first undefine those + fid.write("unset -f $(env | grep '%=' | cut -f1 -d'%' | sed 's/BASH_FUNC_//g')\n") + fid.write("unset $(env | cut -f1 -d=)\n") + # excludes bash functions (environment variables ending with %) fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in sorted(env.items()) if not key.endswith('%')) + '\n') - # unset environment variables in current environment if they're not defined in environment used to run command - fid.write('\n'.join(f'unset {key}' for key in os.environ if key not in env and not key.endswith('%'))) - fid.write('\n\nPS1="eb-shell> "') # also change to working directory (to ensure that working directory is correct for interactive bash shell) diff --git a/test/framework/run.py b/test/framework/run.py index bc32a877a9..7668ee00b1 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -209,9 +209,10 @@ def test_run_shell_cmd_basic(self): self.assertIn("history -s 'echo hello'", env_script_txt) with self.mocked_stdout_stderr(): - res = run_shell_cmd(f"source {env_script}; echo $FOOBAR; history") + res = run_shell_cmd(f"source {env_script}; echo $USER; echo $FOOBAR; history") self.assertEqual(res.exit_code, 0) - self.assertTrue(res.output.startswith('foobar\n')) + user = os.getenv('USER') + self.assertTrue(res.output.startswith(f'{user}\nfoobar\n')) self.assertTrue(res.output.endswith("echo hello\n")) # check on cmd.sh script that can be used to create interactive shell environment for command @@ -268,18 +269,23 @@ def test_run_shell_cmd_env(self): self.assertEqual(len(paths), 1) cmd_tmpdir = paths[0] + # set environment variable in current environment, + # this should not be set in shell environment produced by scripts + os.environ['TEST123'] = 'test123' + env_script = os.path.join(cmd_tmpdir, 'env.sh') self.assertExists(env_script) env_script_txt = read_file(env_script) - self.assertTrue(env_script_txt.startswith('export FOOBAR=foobar\nexport PATH')) + self.assertTrue(env_script_txt.startswith('unset -f $(')) + self.assertIn('\nexport FOOBAR=foobar\nexport PATH', env_script_txt) cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh') self.assertExists(cmd_script) with self.mocked_stdout_stderr(): - res = run_shell_cmd(f"{cmd_script} -c 'echo $FOOBAR'", fail_on_error=False) + res = run_shell_cmd(f"{cmd_script} -c 'echo $FOOBAR; echo TEST123:$TEST123'", fail_on_error=False) self.assertEqual(res.exit_code, 0) - self.assertTrue(res.output.endswith('\nfoobar\n')) + self.assertTrue(res.output.endswith('\nfoobar\nTEST123:\n')) def test_fileprefix_from_cmd(self): """test simplifications from fileprefix_from_cmd.""" From 1bdc93965d123366c7dc5970ddf526f9e410a7f1 Mon Sep 17 00:00:00 2001 From: Jasper Grimm <65227842+jfgrimm@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:11:52 +0100 Subject: [PATCH 329/430] Apply suggestions from code review Co-authored-by: Kenneth Hoste --- easybuild/framework/easyconfig/parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index 6860eb87fd..4787bd97a4 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -55,9 +55,9 @@ 'deps': 'dependencies', 'doc_paths': 'docpaths', 'doc_urls': 'docurls', - 'dont_create_install_dir': 'dontcreateinstalldir', + 'do_not_create_install_dir': 'dontcreateinstalldir', 'exts_class_map': 'exts_classmap', - 'exts_default_class': 'exts_default_class', + 'exts_default_class': 'exts_defaultclass', 'exts_default_opts': 'exts_default_options', 'hidden_deps': 'hiddendependencies', 'include_modulepath_exts': 'include_modpath_extensions', @@ -74,12 +74,12 @@ 'env_mod_load_msg': 'modloadmsg', 'env_mod_lua_footer': 'modluafooter', 'env_mod_tcl_footer': 'modtclfooter', - 'env_mod_class': 'moduleclass', + 'env_mod_category': 'moduleclass', 'env_mod_depends_on': 'module_depends_on', 'env_mod_force_unload': 'moduleforceunload', 'env_mod_load_no_conflict': 'moduleloadnoconflict', 'env_mod_unload_msg': 'modunloadmsg', - 'only_toolchain_mod_env': 'onlytcmod', + 'only_toolchain_env_mod': 'onlytcmod', 'os_deps': 'osdependencies', 'post_install_cmds': 'postinstallcmds', 'post_install_msgs': 'postinstallmsgs', From 4f7a4e44ce51baac570f7392ed3261c4036052a5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Jun 2024 14:00:49 +0200 Subject: [PATCH 330/430] improve error message on version mismatch when using --from-pr + rename UNKNOWN_VERSION to UNKNOWN_EASYBLOCKS_VERSION --- easybuild/main.py | 5 +++-- easybuild/tools/github.py | 4 ++-- easybuild/tools/version.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 00bc362666..451152f322 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -83,7 +83,8 @@ from easybuild.tools.repository.repository import init_repository from easybuild.tools.systemtools import check_easybuild_deps from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state -from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION, UNKNOWN_VERSION, different_major_versions +from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION, UNKNOWN_EASYBLOCKS_VERSION +from easybuild.tools.version import different_major_versions _log = None @@ -620,7 +621,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr from_pr_list, tweaked_ecs_paths) = cfg_settings # compare running Framework and EasyBlocks versions - if EASYBLOCKS_VERSION == UNKNOWN_VERSION: + if EASYBLOCKS_VERSION == UNKNOWN_EASYBLOCKS_VERSION: # most likely reason is running framework unit tests with no easyblocks installation # so log a warning, to avoid test related issues _log.warning("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework") diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 0b5a2ff171..6a8401d7ae 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -594,8 +594,8 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi ver = _get_version_for_repo(os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py')) if different_major_versions(FRAMEWORK_VERSION, ver): - raise EasyBuildError("Framework (%s) is a different major version than PR target (%s)." % (FRAMEWORK_VERSION, - ver)) + raise EasyBuildError("Framework (%s) is a different major version than used in %s/%s PR #%s (%s)", + FRAMEWORK_VERSION, github_account, github_repo, pr, ver) return files diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 3b0c45d2bf..e29f6f0b29 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -47,7 +47,7 @@ # This causes problems further up the dependency chain... VERSION = LooseVersion('4.9.2.dev0') UNKNOWN = 'UNKNOWN' -UNKNOWN_VERSION = '0.0.UNKNOWN.EASYBLOCKS' +UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' def get_git_revision(): @@ -88,7 +88,7 @@ def get_git_revision(): try: from easybuild.easyblocks import VERBOSE_VERSION as EASYBLOCKS_VERSION except Exception: - EASYBLOCKS_VERSION = UNKNOWN_VERSION # make sure it is smaller then anything + EASYBLOCKS_VERSION = UNKNOWN_EASYBLOCKS_VERSION # make sure it is smaller then anything def this_is_easybuild(): From aa7cb7ce52a6f2dec9a45abec60f1a18eb701639 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Jun 2024 19:18:47 +0200 Subject: [PATCH 331/430] extend test_run_shell_cmd_with_hooks to check that run_shell_cmd pre hook is triggered sufficiently early --- test/framework/run.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/framework/run.py b/test/framework/run.py index 23fdb562d3..4e8c3816c3 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1854,6 +1854,32 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ]) self.assertEqual(stdout, expected_stdout) + # also check in dry run mode, to verify that pre-run_shell_cmd hook is triggered sufficiently early + update_build_option('extended_dry_run', True) + + with self.mocked_stdout_stderr(): + run_shell_cmd("make") + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook 'make' in %s" % cwd, + ' running shell command "echo make"', + ' (in %s)' % cwd, + '', + ]) + self.assertEqual(stdout, expected_stdout) + + # also check with trace output enabled + update_build_option('extended_dry_run', False) + update_build_option('trace', True) + + with self.mocked_stdout_stderr(): + run_shell_cmd("make") + stdout = self.get_stdout() + + regex = re.compile('>> running shell command:\n\techo make', re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + def suite(): """ returns all the testcases in this module """ From 70a672ad427288ac4498128bdb4811fd1010622d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Jun 2024 20:36:24 +0200 Subject: [PATCH 332/430] include alternative names of easyconfig parameters in output of --avail-easyconfig-params --- easybuild/tools/docs.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index ba57976f90..c0d782d946 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -50,7 +50,7 @@ from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, process_easyconfig from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.parser import EasyConfigParser +from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, EasyConfigParser from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_DYNAMIC from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_NAMES_EASYCONFIG from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_LOWER, TEMPLATE_NAMES_LOWER_TEMPLATE @@ -315,7 +315,7 @@ def avail_easyconfig_licenses_md(): return '\n'.join(doc) -def avail_easyconfig_params_md(title, grouped_params): +def avail_easyconfig_params_md(title, grouped_params, alternative_params): """ Compose overview of available easyconfig parameters, in MarkDown format. """ @@ -328,13 +328,14 @@ def avail_easyconfig_params_md(title, grouped_params): for grpname in grouped_params: # group section title title = "%s%s parameters" % (grpname[0].upper(), grpname[1:]) - table_titles = ["**Parameter name**", "**Description**", "**Default value**"] + table_titles = ["**Parameter name**", "**Description**", "**Default value**", "**Alternative name**"] keys = sorted(grouped_params[grpname].keys()) values = [grouped_params[grpname][key] for key in keys] table_values = [ ['`%s`' % name for name in keys], # parameter name [x[0].replace('<', '<').replace('>', '>') for x in values], # description - ['`' + str(quote_str(x[1])) + '`' for x in values] # default value + ['`' + str(quote_str(x[1])) + '`' for x in values], # default value + ['`%s`' % alternative_params[name] if name in alternative_params else '' for name in keys], ] doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) @@ -343,7 +344,7 @@ def avail_easyconfig_params_md(title, grouped_params): return '\n'.join(doc) -def avail_easyconfig_params_rst(title, grouped_params): +def avail_easyconfig_params_rst(title, grouped_params, alternative_params): """ Compose overview of available easyconfig parameters, in RST format. """ @@ -357,13 +358,14 @@ def avail_easyconfig_params_rst(title, grouped_params): for grpname in grouped_params: # group section title title = "%s parameters" % grpname - table_titles = ["**Parameter name**", "**Description**", "**Default value**"] + table_titles = ["**Parameter name**", "**Description**", "**Default value**", "**Alternative name**"] keys = sorted(grouped_params[grpname].keys()) values = [grouped_params[grpname][key] for key in keys] table_values = [ ['``%s``' % name for name in keys], # parameter name [x[0] for x in values], # description - [str(quote_str(x[1])) for x in values] # default value + [str(quote_str(x[1])) for x in values], # default value + ['``%s``' % alternative_params[name] if name in alternative_params else '' for name in keys], ] doc.extend(rst_title_and_table(title, table_titles, table_values)) @@ -372,14 +374,14 @@ def avail_easyconfig_params_rst(title, grouped_params): return '\n'.join(doc) -def avail_easyconfig_params_json(): +def avail_easyconfig_params_json(*args): """ Compose overview of available easyconfig parameters, in json format. """ raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json") -def avail_easyconfig_params_txt(title, grouped_params): +def avail_easyconfig_params_txt(title, grouped_params, alternative_params): """ Compose overview of available easyconfig parameters, in plain text format. """ @@ -399,7 +401,17 @@ def avail_easyconfig_params_txt(title, grouped_params): # line by parameter for name, (descr, dflt) in sorted(grouped_params[grpname].items()): - doc.append("{0:<{nw}} {1:} [default: {2:}]".format(name, descr, str(quote_str(dflt)), nw=nw)) + line = ' '.join([ + '{0:<{nw}} ', + '{1:}', + '[default: {2:}]', + ]).format(name, descr, str(quote_str(dflt)), nw=nw) + + alternative = alternative_params.get(name) + if alternative: + line += ' {alternative: %s}' % alternative + + doc.append(line) doc.append('') return '\n'.join(doc) @@ -418,6 +430,9 @@ def avail_easyconfig_params(easyblock, output_format=FORMAT_TXT): extra_params = app.extra_options() params.update(extra_params) + # reverse mapping of alternative easyconfig parameter names + alternative_params = {v: k for k, v in ALTERNATE_PARAMETERS.items()} + # compose title title = "Available easyconfig parameters" if extra_params: @@ -443,7 +458,7 @@ def avail_easyconfig_params(easyblock, output_format=FORMAT_TXT): del grouped_params[grpname] # compose output, according to specified format (txt, rst, ...) - return generate_doc('avail_easyconfig_params_%s' % output_format, [title, grouped_params]) + return generate_doc('avail_easyconfig_params_%s' % output_format, [title, grouped_params, alternative_params]) def avail_easyconfig_templates(output_format=FORMAT_TXT): From 8e845a88749d5ea098ee26140593542b1565b40c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Jun 2024 21:01:46 +0200 Subject: [PATCH 333/430] set usedforsecurity to False when calling hashlib.md5 with Python >= 3.9 --- easybuild/tools/filetools.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 64f9e07ac5..73eac2241f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -123,11 +123,23 @@ CHECKSUM_TYPE_SHA256 = 'sha256' DEFAULT_CHECKSUM = CHECKSUM_TYPE_SHA256 + +def _hashlib_md5(): + """ + Wrapper function for hashlib.md5, + to set usedforsecurity to False when supported (Python >= 3.9) + """ + kwargs = {} + if sys.version_info[0] >= 3 and sys.version_info[1] >= 9: + kwargs = {'usedforsecurity': False} + return hashlib.md5(**kwargs) + + # map of checksum types to checksum functions CHECKSUM_FUNCTIONS = { 'adler32': lambda p: calc_block_checksum(p, ZlibChecksum(zlib.adler32)), 'crc32': lambda p: calc_block_checksum(p, ZlibChecksum(zlib.crc32)), - CHECKSUM_TYPE_MD5: lambda p: calc_block_checksum(p, hashlib.md5()), + CHECKSUM_TYPE_MD5: lambda p: calc_block_checksum(p, _hashlib_md5()), 'sha1': lambda p: calc_block_checksum(p, hashlib.sha1()), CHECKSUM_TYPE_SHA256: lambda p: calc_block_checksum(p, hashlib.sha256()), 'sha512': lambda p: calc_block_checksum(p, hashlib.sha512()), From 4bbba2613c7852c1a04c4f12f6d3af8738483395 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Jun 2024 20:43:21 +0200 Subject: [PATCH 334/430] also consider $CRAY_PE_LIBSCI_PREFIX_DIR to determine installation prefix for cray-libsci (fixes #4536) --- easybuild/toolchains/linalg/libsci.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py index 07ea64ed82..6d7e93c16d 100644 --- a/easybuild/toolchains/linalg/libsci.py +++ b/easybuild/toolchains/linalg/libsci.py @@ -65,13 +65,20 @@ def _get_software_root(self, name, required=True): """Get install prefix for specified software name; special treatment for Cray modules.""" if name == 'cray-libsci': # Cray-provided LibSci module - env_var = 'CRAY_LIBSCI_PREFIX_DIR' - root = os.getenv(env_var, None) + root = None + # consider both $CRAY_LIBSCI_PREFIX_DIR and $CRAY_PE_LIBSCI_PREFIX_DIR, + # cfr. https://github.com/easybuilders/easybuild-framework/issues/4536 + env_vars = ('CRAY_LIBSCI_PREFIX_DIR', 'CRAY_PE_LIBSCI_PREFIX_DIR') + for env_var in env_vars: + root = os.getenv(env_var, None) + if root is not None: + self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + break + if root is None: if required: - raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) - else: - self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + env_vars_str = ', '.join('$' + e for e in env_vars) + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_vars_str) else: root = super(LibSci, self)._get_software_root(name, required=required) From 94976eb7d443517ba802c6e969e881e7de7a8c4b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 7 Jun 2024 12:32:57 +0200 Subject: [PATCH 335/430] symlink downloaded repo at specified commit when using --from-commit so easyconfigs for dependencies are found --- easybuild/tools/github.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ae9b78b5ed..4fd4486a1a 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -612,6 +612,13 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, if github_repo is None: github_repo = GITHUB_EASYCONFIGS_REPO + if github_repo == GITHUB_EASYCONFIGS_REPO: + easybuild_subdir = os.path.join('easybuild', 'easyconfigs') + elif github_repo == GITHUB_EASYBLOCKS_REPO: + easybuild_subdir = os.path.join('easybuild', 'easyblocks') + else: + raise EasyBuildError("Unknown repo: %s", github_repo) + if path is None: if github_repo == GITHUB_EASYCONFIGS_REPO: extra_ec_paths = build_option('extra_ec_paths') @@ -655,6 +662,12 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, else: raise EasyBuildError("Unknown repo: %s" % github_repo) + # symlink subdirectories of 'easybuild/easy{blocks,configs}' into path that gets added to robot search path + mkdir(path, parents=True) + dirpath = os.path.join(repo_commit, easybuild_subdir) + for subdir in os.listdir(dirpath): + symlink(os.path.join(dirpath, subdir), os.path.join(path, subdir)) + # copy specified files to directory where they're expected to be found file_paths = [] for file in files: From 76c511405eaad786cda798587fe086666e068687 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 7 Jun 2024 13:09:21 +0200 Subject: [PATCH 336/430] enhance test for --from-commit to also check whether easyconfigs required for dependencies are found + fix check in test_github_from_pr --- test/framework/options.py | 50 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 3deaf8cf9c..6263d8186f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1908,7 +1908,7 @@ def test_github_from_pr(self): # make sure that *only* these modules are listed, no others regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) - self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules)) pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr6424') regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) @@ -1943,12 +1943,12 @@ def test_github_from_pr(self): # make sure that *only* these modules are listed, no others regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) - self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules)) for pr in ('12150', '12366'): pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr%s' % pr) regex = re.compile(r"Extended list of robot search paths with .*%s.*:" % pr_tmpdir, re.M) - self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + self.assertTrue(regex.search(outtxt), "Found pattern '%s' in: %s" % (regex.pattern, outtxt)) except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) @@ -2125,6 +2125,50 @@ def test_from_commit(self): print("Ignoring URLError '%s' in test_from_commit" % err) shutil.rmtree(tmpdir) + easyblock_template = '\n'.join([ + "from easybuild.framework.easyblock import EasyBlock", + "class %s(EasyBlock):", + " pass", + ]) + + # create fake custom easyblock for CMake that is required by easyconfig used in test below + easyblock_file = os.path.join(self.test_prefix, 'easyblocks', 'cmake.py') + write_file(easyblock_file, easyblock_template % 'EB_CMake') + + # also test with an easyconfig that requires additional easyconfigs to resolve dependencies, + # cfr. https://github.com/easybuilders/easybuild-framework/issues/4540; + # using commit that adds CMake-3.18.4.eb (which requires ncurses-6.2.eb), + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/13156 + test_commit = '41eee3fe2e5102f52319481ca8dde16204dab590' + args = [ + '--from-commit=%s' % test_commit, + '--dry-run', + '--tmpdir=%s' % tmpdir, + '--include-easyblocks=' + os.path.join(self.test_prefix, 'easyblocks', '*.py'), + ] + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (tmpdir, 'ncurses/6.2'), + (tmpdir, 'CMake/3.18.4'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + path = '.*%s' % os.path.dirname(path_prefix) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules)) + + pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_commit_%s' % test_commit) + regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + except URLError as err: + print("Ignoring URLError '%s' in test_from_commit" % err) + shutil.rmtree(tmpdir) + # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... def test_xxx_include_easyblocks_from_commit(self): From 171fdb81465d362fad0d223981edcb914c268fb2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 7 Jun 2024 17:07:44 +0200 Subject: [PATCH 337/430] code style cleanup in get_software_libdir + enhance test --- easybuild/tools/modules.py | 15 +++++++++------ test/framework/modules.py | 13 +++++++++++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index a24ddaa369..7318be4f4f 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1706,13 +1706,16 @@ def get_software_libdir(name, only_one=True, fs=None): if len(res) == 1: res = res[0] else: - if fs is None: - # check if only one (exactly) has libraries + if fs is None and len(res) == 2: + # if both lib and lib64 were found, check if only one (exactly) has libraries; # this is needed for software with library archives in lib64 but other files/directories in lib - lib_glob = ["*.%s" % ext for ext in ["a", get_shared_lib_ext()]] - haslibs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res] - if haslibs[0] != haslibs[1]: - return res[int(not haslibs[0])] + lib_glob = ['*.%s' % ext for ext in ['a', get_shared_lib_ext()]] + has_libs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res] + if has_libs[0] and not has_libs[1]: + return res[0] + elif has_libs[1] and not has_libs[0]: + return res[1] + raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s", name, root, ', '.join(res)) return res diff --git a/test/framework/modules.py b/test/framework/modules.py index bb11802db7..195bf0339c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -51,6 +51,7 @@ from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext # number of modules included for testing purposes @@ -678,11 +679,19 @@ def test_get_software_root_version_libdir(self): root = os.path.join(tmpdir, name) mkdir(os.path.join(root, 'lib64')) os.environ['EBROOT%s' % env_var_name] = root - write_file(os.path.join(root, 'lib', 'foo.a'), 'foo') + write_file(os.path.join(root, 'lib', 'libfoo.a'), 'foo') self.assertEqual(get_software_libdir(name), 'lib') + remove_file(os.path.join(root, 'lib', 'libfoo.a')) + + # also check vice versa with *shared* library in lib64 + shlib_ext = get_shared_lib_ext() + write_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext), 'foo') + self.assertEqual(get_software_libdir(name), 'lib64') + + remove_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext)) + # check expected result of get_software_libdir with multiple lib subdirs - remove_file(os.path.join(root, 'lib', 'foo.a')) self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name) self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64']) From d4fdbae7a66470a891004feaadc2ce9cae26d65d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 7 Jun 2024 21:39:05 +0200 Subject: [PATCH 338/430] don't try to determine repo version when file that contains version doesn't exist --- easybuild/tools/github.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 9a6bc985b8..6d4a1300fc 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -589,13 +589,19 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi raise EasyBuildError("Couldn't find path to patched file %s", full_path) if github_repo == GITHUB_EASYCONFIGS_REPO: - ver = _get_version_for_repo(os.path.join(final_path, 'setup.py')) + ver_file = os.path.join(final_path, 'setup.py') elif github_repo == GITHUB_EASYBLOCKS_REPO: - ver = _get_version_for_repo(os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py')) - - if different_major_versions(FRAMEWORK_VERSION, ver): - raise EasyBuildError("Framework (%s) is a different major version than used in %s/%s PR #%s (%s)", - FRAMEWORK_VERSION, github_account, github_repo, pr, ver) + ver_file = os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py') + else: + raise EasyBuildError("Don't know how to determine version for repo %s", github_repo) + + # take into account that the file we need to determine repo version may not be available, + # for example when a closed PR is used (since then we only download files patched by the PR) + if os.path.exists(ver_file): + ver = _get_version_for_repo(ver_file) + if different_major_versions(FRAMEWORK_VERSION, ver): + raise EasyBuildError("Framework (%s) is a different major version than used in %s/%s PR #%s (%s)", + FRAMEWORK_VERSION, github_account, github_repo, pr, ver) return files From 06c45c3869d2604d22d1dec99c2f89848867ae4a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Jun 2024 12:39:06 +0200 Subject: [PATCH 339/430] split out tests that delete cwd from test_run_shell_cmd_with_hooks into their own test --- easybuild/tools/run.py | 31 ++++++++++------- test/framework/run.py | 78 ++++++++++++++++++++++++++---------------- 2 files changed, 66 insertions(+), 43 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 73638b835a..24715b58eb 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -361,11 +361,14 @@ def to_cmd_str(cmd): if qa_wait_patterns is None: qa_wait_patterns = [] + # keep path to current working dir in case we need to come back to it + try: + initial_work_dir = os.getcwd() + except FileNotFoundError: + raise EasyBuildError(CWD_NOTFOUND_ERROR) + if work_dir is None: - try: - work_dir = os.getcwd() - except FileNotFoundError: - raise EasyBuildError(CWD_NOTFOUND_ERROR) + work_dir = initial_work_dir if with_hooks: hooks = load_hooks(build_option('hooks')) @@ -559,19 +562,21 @@ def to_cmd_str(cmd): raise_run_shell_cmd_error(res) # check that we still are in a sane environment after command execution - # safeguard against commands that leave behind the system in a borked state + # safeguard against commands that deleted the work dir or missbehaving filesystems try: os.getcwd() except FileNotFoundError: + _log.warning( + f"Shell command `{cmd_str}` completed successfully but left the system in a unknown working directory. " + f"Changing back to initial working directory: {initial_work_dir}" + ) try: - warn_msg = ( - f"Shell command `{cmd_str}` completed successfully but left system in a broken state. " - f"Changing back to initial working directory: {res.work_dir}" - ) - _log.warning(warn_msg) - os.chdir(res.work_dir) + os.chdir(initial_work_dir) except OSError as err: - raise EasyBuildError(f"Failed to return to {res.work_dir} after executing command `{cmd_str}`: {err}") + raise EasyBuildError(f"Failed to return to {initial_work_dir} after executing command `{cmd_str}`: {err}") + else: + if not os.path.isdir(work_dir): + work_dir = initial_work_dir if with_hooks: run_hook_kwargs = { @@ -579,7 +584,7 @@ def to_cmd_str(cmd): 'interactive': interactive, 'output': res.output, 'stderr': res.stderr, - 'work_dir': res.work_dir, + 'work_dir': work_dir, } run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) diff --git a/test/framework/run.py b/test/framework/run.py index b464a52c2c..ab0d67e289 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1976,54 +1976,72 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): regex = re.compile('>> running shell command:\n\techo make', re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - # Test commands that destroy directories inside initial working directory + def test_run_shell_cmd_delete_cwd(self): + """ + Test commands that destroy directories inside initial working directory + """ workdir = os.path.join(self.test_prefix, 'workdir') sub_workdir = os.path.join(workdir, 'subworkdir') # 1. test destruction of CWD which is a subdirectory inside original working directory - cmd_workdir_rm = "echo 'Command that jumps to subdir and removes it' && " - cmd_workdir_rm += f"cd {sub_workdir} && pwd && rm -rf {sub_workdir} && " - cmd_workdir_rm += "echo 'Working sub-directory removed.'" - expected_stdout_patterns = [ - rf"pre-run hook '{cmd_workdir_rm}' in {workdir}", - (rf"post-run hook '{cmd_workdir_rm}' \(exit code: 0, " - rf"output: 'Command that jumps to subdir.*\n{sub_workdir}\nWorking sub-directory removed\..*'\)"), - ] - expected_stdout_patterns = [re.compile(pattern, re.S) for pattern in expected_stdout_patterns] + cmd_subworkdir_rm = ( + "echo 'Command that jumps to subdir and removes it' && " + f"cd {sub_workdir} && pwd && rm -rf {sub_workdir} && " + "echo 'Working sub-directory removed.'" + ) # 1.a. in a robust system + expected_output = ( + "Command that jumps to subdir and removes it\n" + f"{sub_workdir}\n" + "Working sub-directory removed.\n" + ) + mkdir(sub_workdir, parents=True) with self.mocked_stdout_stderr(): - run_shell_cmd(cmd_workdir_rm, work_dir=workdir) - stdout = self.get_stdout() + res = run_shell_cmd(cmd_subworkdir_rm, work_dir=workdir) - for regex in expected_stdout_patterns: - self.assertTrue(regex.search(stdout), f"Pattern '{regex.pattern}' should be found in: {stdout}") + self.assertEqual(res.cmd, cmd_subworkdir_rm) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, expected_output) + self.assertEqual(res.stderr, None) + self.assertEqual(res.work_dir, workdir) - # 1.b. in a flaky system that ends up in a broken environment after execution + # 1.b. in a flaky system that ends up in an unknown CWD after execution mkdir(sub_workdir, parents=True) + fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') + os.close(fd) + with self.mocked_stdout_stderr(): with mock.patch('os.getcwd') as mock_getcwd: - mock_getcwd.side_effect = FileNotFoundError() - run_shell_cmd(cmd_workdir_rm, work_dir=workdir) - stdout = self.get_stdout() + mock_getcwd.side_effect = [ + workdir, + FileNotFoundError(), + ] + init_logging(logfile, silent=True) + res = run_shell_cmd(cmd_subworkdir_rm, work_dir=workdir) + stop_logging(logfile) - for regex in expected_stdout_patterns: - self.assertTrue(regex.search(stdout), f"Pattern '{regex.pattern}' should be found in: {stdout}") + self.assertEqual(res.cmd, cmd_subworkdir_rm) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, expected_output) + self.assertEqual(res.stderr, None) + self.assertEqual(res.work_dir, workdir) + + expected_warning = f"Changing back to initial working directory: {workdir}\n" + logtxt = read_file(logfile) + self.assertTrue(logtxt.endswith(expected_warning)) # 2. test destruction of CWD which is main working directory passed to run_shell_cmd - cmd_workdir_rm = "echo 'Command that removes working directory' && pwd && " - cmd_workdir_rm += f"rm -rf {workdir} && echo 'Working directory removed.'" - expected_stdout_patterns = [ - rf"pre-run hook '{cmd_workdir_rm}' in {workdir}", - (rf"post-run hook '{cmd_workdir_rm}' \(exit code: 0, " - rf"output: 'Command that removes working.*\n{workdir}\nWorking directory removed\..*'\)"), - ] - expected_stdout_patterns = [re.compile(pattern, re.S) for pattern in expected_stdout_patterns] + cmd_workdir_rm = ( + "echo 'Command that removes working directory' && pwd && " + f"rm -rf {workdir} && echo 'Working directory removed.'" + ) + + error_pattern = rf"Failed to return to {workdir} after executing command" - mkdir(workdir) + mkdir(workdir, parents=True) with self.mocked_stdout_stderr(): - error_pattern = rf"Failed to return to {workdir} after executing command" self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir) From 8d734993305a7d9ac90d353ec7a0f9581b4766f0 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Mon, 10 Jun 2024 12:34:28 +0100 Subject: [PATCH 340/430] fix use of alternate/alternative --- easybuild/framework/easyblock.py | 4 +- easybuild/framework/easyconfig/easyconfig.py | 34 +++++++------- .../easyconfig/format/pyheaderconfigobj.py | 10 ++--- easybuild/framework/easyconfig/parser.py | 6 +-- easybuild/framework/easyconfig/templates.py | 12 ++--- easybuild/framework/easyconfig/tweak.py | 8 ++-- easybuild/toolchains/compiler/craype.py | 2 +- easybuild/tools/filetools.py | 6 +-- easybuild/tools/toolchain/toolchain.py | 2 +- test/framework/easyblock.py | 8 ++-- test/framework/easyconfig.py | 44 +++++++++---------- test/framework/toolchain.py | 2 +- test/framework/toy_build.py | 2 +- 13 files changed, 70 insertions(+), 70 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a002b733b8..3cb237b594 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -912,10 +912,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if PYPI_PKG_URL_PATTERN in fullurl and not is_alt_pypi_url(fullurl): alt_url = derive_alt_pypi_url(fullurl) if alt_url: - _log.debug("Using alternate PyPI URL for %s: %s", fullurl, alt_url) + _log.debug("Using alternative PyPI URL for %s: %s", fullurl, alt_url) fullurl = alt_url else: - _log.debug("Failed to derive alternate PyPI URL for %s, so retaining the original", fullurl) + _log.debug("Failed to derive alternative PyPI URL for %s, so retaining the original", fullurl) if self.dry_run: self.dry_run_msg(" * %s will be downloaded to %s", filename, targetpath) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 096008fc15..a95efabb3a 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -59,9 +59,9 @@ from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION, retrieve_blocks_in_spec from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS, REPLACED_PARAMETERS +from easybuild.framework.easyconfig.parser import ALTERNATIVE_EASYCONFIG_PARAMETERS, DEPRECATED_EASYCONFIG_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATES, DEPRECATED_TEMPLATES, TEMPLATE_CONSTANTS +from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES, TEMPLATE_CONSTANTS from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg @@ -119,11 +119,11 @@ def handle_deprecated_or_replaced_easyconfig_parameters(ec_method): def new_ec_method(self, key, *args, **kwargs): """Check whether any replace easyconfig parameters are still used""" # map deprecated parameters to their replacements, issue deprecation warning(/error) - if key in ALTERNATE_PARAMETERS: - key = ALTERNATE_PARAMETERS[key] - elif key in DEPRECATED_PARAMETERS: + if key in ALTERNATIVE_EASYCONFIG_PARAMETERS: + key = ALTERNATIVE_EASYCONFIG_PARAMETERS[key] + elif key in DEPRECATED_EASYCONFIG_PARAMETERS: depr_key = key - key, ver = DEPRECATED_PARAMETERS[depr_key] + key, ver = DEPRECATED_EASYCONFIG_PARAMETERS[depr_key] _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver) elif key in REPLACED_PARAMETERS: _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') @@ -182,7 +182,7 @@ def triage_easyconfig_params(variables, ec): for key in variables: # validations are skipped, just set in the config - if any(key in d for d in (ec, DEPRECATED_PARAMETERS.keys(), ALTERNATE_PARAMETERS.keys())): + if any(key in d for d in (ec, DEPRECATED_EASYCONFIG_PARAMETERS.keys(), ALTERNATIVE_EASYCONFIG_PARAMETERS.keys())): ec_params[key] = variables[key] _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key])) elif key in REPLACED_PARAMETERS: @@ -661,7 +661,7 @@ def set_keys(self, params): with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config - if any(key in x.keys() for x in (self._config, ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS)): + if any(key in x.keys() for x in (self._config, ALTERNATIVE_EASYCONFIG_PARAMETERS, DEPRECATED_EASYCONFIG_PARAMETERS)): self[key] = params[key] self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) @@ -1998,24 +1998,24 @@ def resolve_template(value, tmpl_dict): try: value = value % tmpl_dict except KeyError: - # check if any alternate and/or deprecated templates resolve + # check if any alternative and/or deprecated templates resolve try: orig_value = value - # map old templates to new values for alternate and deprecated templates + # map old templates to new values for alternative and deprecated templates alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in - ALTERNATE_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} + ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} alt_map2 = {new_tmpl: tmpl_dict[old_tmpl] for (old_tmpl, new_tmpl) in - ALTERNATE_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()} + ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()} depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in - DEPRECATED_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} + DEPRECATED_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} - # try templating with alternate and deprecated templates included + # try templating with alternative and deprecated templates included value = value % {**tmpl_dict, **alt_map, **alt_map2, **depr_map} for old_tmpl, val in depr_map.items(): # check which deprecated templates were replaced, and issue deprecation warnings if old_tmpl in orig_value and val in value: - new_tmpl, ver = DEPRECATED_TEMPLATES[old_tmpl] + new_tmpl, ver = DEPRECATED_EASYCONFIG_TEMPLATES[old_tmpl] _log.deprecated(f"Easyconfig template '{old_tmpl}' is deprecated, use '{new_tmpl}' instead", ver) except KeyError: @@ -2023,8 +2023,8 @@ def resolve_template(value, tmpl_dict): value = raw_value # Undo "%"-escaping for key in tmpl_dict: - if key in DEPRECATED_TEMPLATES: - new_key, ver = DEPRECATED_TEMPLATES[key] + if key in DEPRECATED_EASYCONFIG_TEMPLATES: + new_key, ver = DEPRECATED_EASYCONFIG_TEMPLATES[key] _log.deprecated(f"Easyconfig template '{key}' is deprecated, use '{new_key}' instead", ver) else: diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 9d3189804a..683094d98e 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -39,7 +39,7 @@ from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATE_CONSTANTS, DEPRECATED_TEMPLATE_CONSTANTS +from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS, DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import ConfigObj @@ -91,10 +91,10 @@ def handle_deprecated_constants(method): """Decorator to handle deprecated easyconfig template constants""" def wrapper(self, key, *args, **kwargs): """Check whether any deprecated constants are used""" - alternate = ALTERNATE_TEMPLATE_CONSTANTS - deprecated = DEPRECATED_TEMPLATE_CONSTANTS - if key in alternate: - key = alternate[key] + alternative = ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS + deprecated = DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS + if key in alternative: + key = alternative[key] elif key in deprecated: depr_key = key key, ver = deprecated[depr_key] diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index 4787bd97a4..7ebdff4873 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -42,8 +42,8 @@ from easybuild.tools.filetools import read_file, write_file -# alternate easyconfig parameters, and their non-deprecated equivalents -ALTERNATE_PARAMETERS = { +# alternative easyconfig parameters, and their non-deprecated equivalents +ALTERNATIVE_EASYCONFIG_PARAMETERS = { # : , 'build_deps': 'builddependencies', 'build_in_install_dir': 'buildininstalldir', @@ -101,7 +101,7 @@ } # deprecated easyconfig parameters, and their replacements -DEPRECATED_PARAMETERS = { +DEPRECATED_EASYCONFIG_PARAMETERS = { # : (, ), } diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 737b105915..d72336b3ef 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -161,8 +161,8 @@ ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'), ] -# alternate templates, and their equivalents -ALTERNATE_TEMPLATES = { +# alternative templates, and their equivalents +ALTERNATIVE_EASYCONFIG_TEMPLATES = { # : , 'build_dir': 'builddir', 'cuda_cc_comma_sep': 'cuda_compute_capabilities', @@ -197,12 +197,12 @@ } # deprecated templates, and their replacements -DEPRECATED_TEMPLATES = { +DEPRECATED_EASYCONFIG_TEMPLATES = { # : (, ), } -# alternate template constants, and their equivalents -ALTERNATE_TEMPLATE_CONSTANTS = { +# alternative template constants, and their equivalents +ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS = { # : , 'APACHE_URL': 'APACHE_SOURCE', 'BITBUCKET_GET_URL': 'BITBUCKET_SOURCE', @@ -244,7 +244,7 @@ } # deprecated template constants, and their replacements -DEPRECATED_TEMPLATE_CONSTANTS = { +DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS = { # : (, ), } diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 3ec3c342ea..2a88e56d6a 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1236,7 +1236,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin (highest_version_ignoring_versionsuffix is not None and highest_version is not None and LooseVersion(highest_version_ignoring_versionsuffix) > LooseVersion(highest_version)) - exclude_alternate_versionsuffixes = False + exclude_alternative_versionsuffixes = False if ignored_versionsuffix_greater: if ignore_versionsuffixes: highest_version = highest_version_ignoring_versionsuffix @@ -1247,11 +1247,11 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin dep['name'], versionsuffix, [d['path'] for d in potential_version_mappings if d['version'] == highest_version_ignoring_versionsuffix]) # exclude candidates with a different versionsuffix - exclude_alternate_versionsuffixes = True + exclude_alternative_versionsuffixes = True else: # If the other version suffixes are not greater, then just ignore them - exclude_alternate_versionsuffixes = True - if exclude_alternate_versionsuffixes: + exclude_alternative_versionsuffixes = True + if exclude_alternative_versionsuffixes: potential_version_mappings = [d for d in potential_version_mappings if d['versionsuffix'] == versionsuffix] if highest_versions_only and highest_version is not None: diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py index 73b6a103e2..1d492c34db 100644 --- a/easybuild/toolchains/compiler/craype.py +++ b/easybuild/toolchains/compiler/craype.py @@ -65,7 +65,7 @@ class CrayPECompiler(Compiler): COMPILER_UNIQUE_OPTS = { 'dynamic': (True, "Generate dynamically linked executable"), - 'mpich-mt': (False, "Directs the driver to link in an alternate version of the Cray-MPICH library which \ + 'mpich-mt': (False, "Directs the driver to link in an alternative version of the Cray-MPICH library which \ provides fine-grained multi-threading support to applications that perform \ MPI operations within threaded regions."), 'optarch': (False, "Enable architecture optimizations"), diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 64f9e07ac5..1ddd790ae0 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -586,11 +586,11 @@ def normalize_path(path): def is_alt_pypi_url(url): - """Determine whether specified URL is already an alternate PyPI URL, i.e. whether it contains a hash.""" + """Determine whether specified URL is already an alternative PyPI URL, i.e. whether it contains a hash.""" # example: .../packages/5b/03/e135b19fadeb9b1ccb45eac9f60ca2dc3afe72d099f6bd84e03cb131f9bf/easybuild-2.7.0.tar.gz alt_url_regex = re.compile('/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/[^/]+$') res = bool(alt_url_regex.search(url)) - _log.debug("Checking whether '%s' is an alternate PyPI URL using pattern '%s'...: %s", + _log.debug("Checking whether '%s' is an alternative PyPI URL using pattern '%s'...: %s", url, alt_url_regex.pattern, res) return res @@ -638,7 +638,7 @@ def handle_starttag(self, tag, attrs): def derive_alt_pypi_url(url): - """Derive alternate PyPI URL for given URL.""" + """Derive alternative PyPI URL for given URL.""" alt_pypi_url = None # example input URL: https://pypi.python.org/packages/source/e/easybuild/easybuild-2.7.0.tar.gz diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 17916de519..3dd76903b6 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1028,7 +1028,7 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None def handle_sysroot(self): """ - Extra stuff to be done when alternate system root is specified via --sysroot EasyBuild configuration option. + Extra stuff to be done when alternative system root is specified via --sysroot EasyBuild configuration option. * Update $PKG_CONFIG_PATH to include sysroot location to pkg-config files (*.pc). """ diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 7451f948b1..30b95c18db 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -310,7 +310,7 @@ def test_make_module_extend_modpath(self): regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) - # Repeat this but using an alternate envvars (instead of $HOME) + # Repeat this but using an alternative envvars (instead of $HOME) list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS'] build_options = { @@ -1813,7 +1813,7 @@ def test_obtain_file(self): res = eb.obtain_file(toy_tarball) self.assertEqual(res, toy_tarball_path) - # finding a file in the alternate location works + # finding a file in the alternative location works with self.mocked_stdout_stderr(): res = eb.obtain_file(toy_tarball, alt_location='alt_toy') self.assertEqual(res, alt_toy_tarball_path) @@ -2720,10 +2720,10 @@ def run_checks(): # no checksum issues self.assertEqual(eb.check_checksums(), []) - # tuple of two alternate SHA256 checksums: OK + # tuple of two alternative SHA256 checksums: OK eb.cfg['checksums'] = [ ( - # two alternate checksums for toy-0.0.tar.gz + # two alternative checksums for toy-0.0.tar.gz 'a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd', '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', ), diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 8152d10905..446536765c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -119,10 +119,10 @@ def setUp(self): github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None - self.orig_easyconfig_DEPRECATED_PARAMETERS = easyconfig.easyconfig.DEPRECATED_PARAMETERS - self.orig_easyconfig_DEPRECATED_TEMPLATES = easyconfig.easyconfig.DEPRECATED_TEMPLATES - self.orig_easyconfig_ALTERNATE_PARAMETERS = easyconfig.easyconfig.ALTERNATE_PARAMETERS - self.orig_easyconfig_ALTERNATE_TEMPLATES = easyconfig.easyconfig.ALTERNATE_TEMPLATES + self.orig_easyconfig_DEPRECATED_EASYCONFIG_PARAMETERS = easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS + self.orig_easyconfig_DEPRECATED_EASYCONFIG_TEMPLATES = easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES + self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_PARAMETERS = easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS + self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_TEMPLATES = easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_TEMPLATES def prep(self): """Prepare for test.""" @@ -142,11 +142,11 @@ def tearDown(self): if os.path.exists(self.eb_file): os.remove(self.eb_file) - # restore orignal values of DEPRECATED_TEMPLATES & co in easyconfig.templates - easyconfig.easyconfig.DEPRECATED_PARAMETERS = self.orig_easyconfig_DEPRECATED_PARAMETERS - easyconfig.easyconfig.DEPRECATED_TEMPLATES = self.orig_easyconfig_DEPRECATED_TEMPLATES - easyconfig.easyconfig.ALTERNATE_PARAMETERS = self.orig_easyconfig_ALTERNATE_PARAMETERS - easyconfig.easyconfig.ALTERNATE_TEMPLATES = self.orig_easyconfig_ALTERNATE_TEMPLATES + # restore orignal values of DEPRECATED_EASYCONFIG_TEMPLATES & co in easyconfig.templates + easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS = self.orig_easyconfig_DEPRECATED_EASYCONFIG_PARAMETERS + easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES = self.orig_easyconfig_DEPRECATED_EASYCONFIG_TEMPLATES + easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS = self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_PARAMETERS + easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_TEMPLATES = self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_TEMPLATES def test_empty(self): """ empty files should not parse! """ @@ -1462,8 +1462,8 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) - def test_template_deprecation_and_alternate(self): - """Test deprecation of (and alternate) templates""" + def test_template_deprecation_and_alternative(self): + """Test deprecation of (and alternative) templates""" self.prep() @@ -1472,13 +1472,13 @@ def test_template_deprecation_and_alternate(self): 'cudaver': ('depr_cuda_ver', '1000000000'), 'start_dir': ('depr_start_dir', '1000000000'), } - easyconfig.easyconfig.DEPRECATED_TEMPLATES = template_test_deprecations + easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES = template_test_deprecations - template_test_alternates = { + template_test_alternatives = { 'installdir': 'alt_install_dir', 'version_major_minor': 'alt_ver_maj_min', } - easyconfig.easyconfig.ALTERNATE_TEMPLATES = template_test_alternates + easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_TEMPLATES = template_test_alternatives tmpl_str = ("cd %(start_dir)s && make %(namelower)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" " && echo %(alt_install_dir)s %(version_major_minor)s") @@ -1495,7 +1495,7 @@ def test_template_deprecation_and_alternate(self): res = resolve_template(tmpl_str, tmpl_dict) stderr = stderr.getvalue() - for tmpl in [*template_test_deprecations.keys(), *template_test_alternates.keys()]: + for tmpl in [*template_test_deprecations.keys(), *template_test_alternatives.keys()]: self.assertNotIn("%(" + tmpl + ")s", res) for old, (new, ver) in template_test_deprecations.items(): @@ -1818,8 +1818,8 @@ def foo(key): self.assertErrorRegex(EasyBuildError, error_regex, foo, key) - def test_alternate_easyconfig_parameters(self): - """Test handling of alternate easyconfig parameters.""" + def test_alternative_easyconfig_parameters(self): + """Test handling of alternative easyconfig parameters.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') @@ -1832,10 +1832,10 @@ def test_alternate_easyconfig_parameters(self): write_file(test_ec, test_ec_txt) # post_install_cmds is not accepted unless it's registered as an alternative easyconfig parameter - easyconfig.easyconfig.ALTERNATE_PARAMETERS = {} + easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS = {} self.assertErrorRegex(EasyBuildError, "post_install_cmds -> postinstallcmds", EasyConfig, test_ec) - easyconfig.easyconfig.ALTERNATE_PARAMETERS = { + easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS = { 'env_mod_class': 'moduleclass', 'post_install_cmds': 'postinstallcmds', } @@ -1849,7 +1849,7 @@ def test_alternate_easyconfig_parameters(self): self.assertEqual(ec['postinstallcmds'], expected) self.assertEqual(ec['post_install_cmds'], expected) - # test setting of easyconfig parameter with original & alternate name + # test setting of easyconfig parameter with original & alternative name ec['moduleclass'] = 'test1' self.assertEqual(ec['moduleclass'], 'test1') self.assertEqual(ec['env_mod_class'], 'test1') @@ -1873,7 +1873,7 @@ def test_deprecated_easyconfig_parameters(self): test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) - easyconfig.easyconfig.DEPRECATED_PARAMETERS = { + easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS = { 'foobar': ('barfoo', '0.0'), # deprecated since forever # won't be actually deprecated for a while; # note that we should map foobarbarfoo to a valid easyconfig parameter here, @@ -1881,7 +1881,7 @@ def test_deprecated_easyconfig_parameters(self): 'foobarbarfoo': ('required_linked_shared_libs', '1000000000'), } - for key, (newkey, depr_ver) in easyconfig.easyconfig.DEPRECATED_PARAMETERS.items(): + for key, (newkey, depr_ver) in easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS.items(): if LooseVersion(depr_ver) <= easybuild.tools.build_log.CURRENT_VERSION: # deprecation error error_regex = "DEPRECATED.*since v%s.*'%s' is deprecated.*use '%s' instead" % (depr_ver, key, newkey) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index eb6bf1d4cd..2ca15d26d6 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -144,7 +144,7 @@ def test_is_system_toolchain(self): def test_toolchain_prepare_sysroot(self): """Test build environment setup done by Toolchain.prepare in case --sysroot is specified.""" - sysroot = os.path.join(self.test_prefix, 'test', 'alternate', 'sysroot') + sysroot = os.path.join(self.test_prefix, 'test', 'alternative', 'sysroot') sysroot_pkgconfig = os.path.join(sysroot, 'usr', 'lib', 'pkgconfig') mkdir(sysroot_pkgconfig, parents=True) init_config(build_options={'sysroot': sysroot}) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 4901b55de7..2978e64b1a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3954,7 +3954,7 @@ def test_toy_build_sanity_check_linked_libs(self): self.assertErrorRegex(EasyBuildError, error_msg, self._test_toy_build, force=False, ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False) - # check behaviour when alternate subdirectories are specified + # check behaviour when alternative subdirectories are specified test_ec_txt = read_file(libtoy_ec) test_ec_txt += "\nbin_lib_subdirs = ['', 'lib', 'lib64']" write_file(test_ec, test_ec_txt) From a31665934340442fceb4ecba9de62ef82528dccf Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Mon, 10 Jun 2024 15:34:51 +0100 Subject: [PATCH 341/430] fix long lines --- easybuild/framework/easyblock.py | 3 ++- easybuild/framework/easyconfig/easyconfig.py | 15 +++++++++------ .../easyconfig/format/pyheaderconfigobj.py | 4 ++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3cb237b594..25de88e202 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -915,7 +915,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No _log.debug("Using alternative PyPI URL for %s: %s", fullurl, alt_url) fullurl = alt_url else: - _log.debug("Failed to derive alternative PyPI URL for %s, so retaining the original", fullurl) + _log.debug("Failed to derive alternative PyPI URL for %s, so retaining the original", + fullurl) if self.dry_run: self.dry_run_msg(" * %s will be downloaded to %s", filename, targetpath) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index a95efabb3a..772057fa6b 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -59,10 +59,11 @@ from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION, retrieve_blocks_in_spec from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.parser import ALTERNATIVE_EASYCONFIG_PARAMETERS, DEPRECATED_EASYCONFIG_PARAMETERS, REPLACED_PARAMETERS -from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES, TEMPLATE_CONSTANTS -from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_DYNAMIC, template_constant_dict +from easybuild.framework.easyconfig.parser import ALTERNATIVE_EASYCONFIG_PARAMETERS, DEPRECATED_EASYCONFIG_PARAMETERS +from easybuild.framework.easyconfig.parser import REPLACED_PARAMETERS, EasyConfigParser +from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig +from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG @@ -182,7 +183,8 @@ def triage_easyconfig_params(variables, ec): for key in variables: # validations are skipped, just set in the config - if any(key in d for d in (ec, DEPRECATED_EASYCONFIG_PARAMETERS.keys(), ALTERNATIVE_EASYCONFIG_PARAMETERS.keys())): + if any(key in d for d in (ec, DEPRECATED_EASYCONFIG_PARAMETERS.keys(), + ALTERNATIVE_EASYCONFIG_PARAMETERS.keys())): ec_params[key] = variables[key] _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key])) elif key in REPLACED_PARAMETERS: @@ -661,7 +663,8 @@ def set_keys(self, params): with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config - if any(key in x.keys() for x in (self._config, ALTERNATIVE_EASYCONFIG_PARAMETERS, DEPRECATED_EASYCONFIG_PARAMETERS)): + if any(key in x.keys() for x in (self._config, ALTERNATIVE_EASYCONFIG_PARAMETERS, + DEPRECATED_EASYCONFIG_PARAMETERS)): self[key] = params[key] self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 683094d98e..48e6516e8c 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -39,8 +39,8 @@ from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS, DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS -from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS +from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS +from easybuild.framework.easyconfig.templates import DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS, TEMPLATE_CONSTANTS from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import ConfigObj from easybuild.tools.systemtools import get_shared_lib_ext From c9cf1aa84a39fbfe5eb91099f71b63aba38fee3a Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Mon, 10 Jun 2024 23:00:29 +0200 Subject: [PATCH 342/430] prepare release notes for EasyBuild v4.9.2 + bump version to 4.9.2 --- RELEASE_NOTES | 23 +++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 07c9756e67..073289de0a 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,29 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.9.2 (11 June 2024) +--------------------- + +update/bugfix release + +- various enhancements, including: + - Improve behavior when using extension with 'nosource:True' (#4506) + - enhance `get_software_libdir` to return `lib` or `lib64` if only one of them contains library files (#4513) + - versions checks to avoid mixing major versions across the EasyBuild components (#4520) + - add support for easyconfig parameter `module_only` (#4537) +- various bug fixes, including: + - Fix typo in patch_step logging (#4505) + - consider both `easybuild-framework*.tar.gz` and `easybuild_framework*.tar.gz` in CI workflows (#4507) + - don't delete existing environment module files when using `--dump-env-script` with `--force` or `--rebuild` (#4512) + - do not run unit tests on Python 3.5 (#4530) + - Fix resolved (template) values in case of failure (#4532) + - also consider `$CRAY_PE_LIBSCI_PREFIX_DIR` to determine installation prefix for cray-libsci (#4551) + - symlink downloaded repo at specified commit when using `--from-commit` so easyconfigs for dependencies are found (#4552) + - don't try to determine repo version when file that contains version doesn't exist (#4553) +- other changes: + - code cleanup in `easyblock.py` (#4519) + + v4.9.1 (5 April 2024) --------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index e29f6f0b29..bbe7f386ad 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.2.dev0') +VERSION = LooseVersion('4.9.2') UNKNOWN = 'UNKNOWN' UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' From 43c9fb4713c9e243b8fb951b4e230e318311a37d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 11 Jun 2024 07:54:22 +0200 Subject: [PATCH 343/430] minor tweaks to 4.9.2 release notes --- RELEASE_NOTES | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 073289de0a..82373cc16a 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -10,21 +10,20 @@ v4.9.2 (11 June 2024) update/bugfix release - various enhancements, including: - - Improve behavior when using extension with 'nosource:True' (#4506) - - enhance `get_software_libdir` to return `lib` or `lib64` if only one of them contains library files (#4513) - - versions checks to avoid mixing major versions across the EasyBuild components (#4520) - - add support for easyconfig parameter `module_only` (#4537) -- various bug fixes, including: - - Fix typo in patch_step logging (#4505) - - consider both `easybuild-framework*.tar.gz` and `easybuild_framework*.tar.gz` in CI workflows (#4507) - - don't delete existing environment module files when using `--dump-env-script` with `--force` or `--rebuild` (#4512) - - do not run unit tests on Python 3.5 (#4530) - - Fix resolved (template) values in case of failure (#4532) - - also consider `$CRAY_PE_LIBSCI_PREFIX_DIR` to determine installation prefix for cray-libsci (#4551) - - symlink downloaded repo at specified commit when using `--from-commit` so easyconfigs for dependencies are found (#4552) - - don't try to determine repo version when file that contains version doesn't exist (#4553) + - improve behavior when using extension which has 'nosource' enabled (#4506) + - enhance 'get_software_libdir' to return 'lib' or 'lib64' if only one of them contains library files (#4513) + - implement versions checks to avoid mixing major versions across the EasyBuild components (#4520, #4553) + - add support for easyconfig parameter 'module_only' (#4537) +- various bug fixes, including: + - fix typo in patch_step logging (#4505) + - consider both 'easybuild-framework*.tar.gz' and 'easybuild_framework*.tar.gz' in CI workflows (#4507) + - don't delete existing environment module files when using '--dump-env-script' with '--force' or '--rebuild' (#4512) + - fix resolved (template) values in case of failure (#4532) + - also consider '$CRAY_PE_LIBSCI_PREFIX_DIR' to determine installation prefix for cray-libsci (#4551) + - symlink downloaded repo at specified commit when using '--from-commit' so easyconfigs for dependencies are found (#4552) - other changes: - - code cleanup in `easyblock.py` (#4519) + - code cleanup in 'easyblock.py' (#4519) + - stop running unit tests on Python 3.5 (#4530) v4.9.1 (5 April 2024) From d9078288216efec344f7c1a9fc7f0fdf11f39d16 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 11 Jun 2024 20:23:34 +0200 Subject: [PATCH 344/430] bump release date for EasyBuild v4.9.2 to 12 June 2024 --- RELEASE_NOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 82373cc16a..5a45be8a56 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,7 +4,7 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. -v4.9.2 (11 June 2024) +v4.9.2 (12 June 2024) --------------------- update/bugfix release From a4cc53b702e97771148261da95a0b67c8dbbc046 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Jun 2024 23:24:58 +0200 Subject: [PATCH 345/430] bump version to 4.9.3dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index bbe7f386ad..0f24a88023 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.2') +VERSION = LooseVersion('4.9.3.dev0') UNKNOWN = 'UNKNOWN' UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' From 02a371918370e0f339830a4b7b623292abb077e7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 10 Jun 2024 09:40:41 +0200 Subject: [PATCH 346/430] Correctly evaluate result for `--dep-graph` The success message was always printed even when dumping the graph failed resulting in no file being created but a message like: > Error: renderer for .pdf is unavailable > Wrote dependency graph for 1 easyconfigs to graph.pdf --- easybuild/framework/easyconfig/tools.py | 26 ++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 6f2a2741ad..1d5d45adb1 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -54,7 +54,7 @@ from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning +from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files @@ -219,10 +219,12 @@ def mk_node_name(spec): if dep in spec['ec'].build_dependencies: dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs) - _dep_graph_dump(dgr, filename) - - if not build_option('silent'): - print("Wrote dependency graph for %d easyconfigs to %s" % (len(specs), filename)) + what = "dependency graph for %d easyconfigs to %s" % (len(specs), filename) + silent = build_option('silent') + if _dep_graph_dump(dgr, filename): + print_msg("Wrote " + what, silent=silent) + else: + print_error("Failed writing " + what, silent=silent) @only_if_module_is_available('pygraph.readwrite.dot', pkgname='python-graph-dot') @@ -232,9 +234,15 @@ def _dep_graph_dump(dgr, filename): dottxt = dot.write(dgr) if os.path.splitext(filename)[-1] == '.dot': # create .dot file - write_file(filename, dottxt) + try: + write_file(filename, dottxt) + except EasyBuildError as e: + print(str(e)) + return False + else: + return True else: - _dep_graph_gv(dottxt, filename) + return _dep_graph_gv(dottxt, filename) @only_if_module_is_available('gv', pkgname='graphviz-python') @@ -242,8 +250,8 @@ def _dep_graph_gv(dottxt, filename): """Render dependency graph to file using graphviz.""" # try and render graph in specified file format gvv = gv.readstring(dottxt) - gv.layout(gvv, 'dot') - gv.render(gvv, os.path.splitext(filename)[-1], filename) + if gv.layout(gvv, 'dot') is not False: + return gv.render(gvv, os.path.splitext(filename)[-1], filename) def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): From 2b7fb6dd903b7da9a7ead29226cca4fefaf74ff8 Mon Sep 17 00:00:00 2001 From: Jasper Grimm <65227842+jfgrimm@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:44:27 +0100 Subject: [PATCH 347/430] use alternative instead of alternate --- easybuild/tools/docs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index c0d782d946..4bbdefa66b 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -50,7 +50,7 @@ from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, process_easyconfig from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, EasyConfigParser +from easybuild.framework.easyconfig.parser import ALTERNATIVE_EASYCONFIG_PARAMETERS, EasyConfigParser from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_DYNAMIC from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_NAMES_EASYCONFIG from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_LOWER, TEMPLATE_NAMES_LOWER_TEMPLATE @@ -431,7 +431,7 @@ def avail_easyconfig_params(easyblock, output_format=FORMAT_TXT): params.update(extra_params) # reverse mapping of alternative easyconfig parameter names - alternative_params = {v: k for k, v in ALTERNATE_PARAMETERS.items()} + alternative_params = {v: k for k, v in ALTERNATIVE_EASYCONFIG_PARAMETERS.items()} # compose title title = "Available easyconfig parameters" From d45b4e5a0f5b28b2e4363c680b6e3f96eee5a9c4 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 18 Jun 2024 11:18:12 +0200 Subject: [PATCH 348/430] fix typo in log message of easybuild.tools.run Co-authored-by: Kenneth Hoste --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 24715b58eb..57975abfeb 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -567,7 +567,7 @@ def to_cmd_str(cmd): os.getcwd() except FileNotFoundError: _log.warning( - f"Shell command `{cmd_str}` completed successfully but left the system in a unknown working directory. " + f"Shell command `{cmd_str}` completed successfully but left the system in an unknown working directory. " f"Changing back to initial working directory: {initial_work_dir}" ) try: From 38ca67d66427b967865304a6f9876b0e0d58df5b Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 19 Jun 2024 10:08:28 +0200 Subject: [PATCH 349/430] revert fallback of work dir passed to post_run_shell_cmd in case of non-existent cwd --- easybuild/tools/run.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 57975abfeb..fbfb36dc74 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -574,9 +574,6 @@ def to_cmd_str(cmd): os.chdir(initial_work_dir) except OSError as err: raise EasyBuildError(f"Failed to return to {initial_work_dir} after executing command `{cmd_str}`: {err}") - else: - if not os.path.isdir(work_dir): - work_dir = initial_work_dir if with_hooks: run_hook_kwargs = { @@ -584,7 +581,7 @@ def to_cmd_str(cmd): 'interactive': interactive, 'output': res.output, 'stderr': res.stderr, - 'work_dir': work_dir, + 'work_dir': res.work_dir, } run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) From e85b114a488b45b560d7ab5576b61fbd79b35d01 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 19 Jun 2024 14:00:45 +0200 Subject: [PATCH 350/430] avoid trace output produced by main in test_skip --- test/framework/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 04ed4e6d28..c9c92663e6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -386,7 +386,8 @@ def test_skip(self): ] test_ec_txt += "\nmodule_only = True\n" write_file(test_ec, test_ec_txt) - self.eb_main(args, do_build=True, raise_error=True) + with self.mocked_stdout_stderr(): + self.eb_main(args, do_build=True, raise_error=True) self.assertEqual(len(glob.glob(toy_mod_glob)), 1) From 467afd54dc5844586dd07467e5f16e53f7b9ece7 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 24 Jun 2024 13:10:54 +0000 Subject: [PATCH 351/430] Adding gmpflf --- easybuild/toolchains/gmpflf.py | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 easybuild/toolchains/gmpflf.py diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py new file mode 100644 index 0000000000..91b5aec135 --- /dev/null +++ b/easybuild/toolchains/gmpflf.py @@ -0,0 +1,47 @@ +## +# Copyright 2013-2024 Ghent University +# +# This file is triple-licensed under GPLv2 (see below), MIT, and +# BSD three-clause licenses. +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmpflf compiler toolchain (includes GCC, MPICH2, OpenBLAS, LAPACK, ScaLAPACK and FFTW). + +Authors: + +* Richard Topouchian (University of Bergen) +""" +from easybuild.toolchains.gmpich import Gmpich +from easybuild.toolchains.gfbf import Gfbf +from easybuild.toolchains.golf import Golf +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS +from easybuild.toolchains.linalg.openblas import OpenBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Gmpflf(Gmpich, OpenBLAS, FlexiBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, MPICH, OpenBLAS, ScaLAPACK and FFTW.""" + NAME = 'gmpflf' + SUBTOOLCHAIN = [Gmpich.NAME, Golf.NAME, Gfbf.NAME] From f8a192cb0b5fbb1071d24837ca0ff1010574bc7b Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 24 Jun 2024 16:17:01 +0200 Subject: [PATCH 352/430] Update gmpflf.py --- easybuild/toolchains/gmpflf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py index 91b5aec135..f040a161da 100644 --- a/easybuild/toolchains/gmpflf.py +++ b/easybuild/toolchains/gmpflf.py @@ -26,7 +26,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for gmpflf compiler toolchain (includes GCC, MPICH2, OpenBLAS, LAPACK, ScaLAPACK and FFTW). +EasyBuild support for gmpflf compiler toolchain (includes GCC, MPICH, FlexiBLAS, LAPACK, ScaLAPACK and FFTW). Authors: From 9da5c353dff77d97425293fc1d23ccce0e53e98e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 27 Jun 2024 14:05:31 +0000 Subject: [PATCH 353/430] Set oneapi_fortran=True by default for Intel 2024.0.0+ ifort is deprecated, so default to ifx --- .../toolchains/compiler/intel_compilers.py | 6 ++- .../modules/intel-compilers/2024.0.0 | 37 +++++++++++++++++++ test/framework/toolchain.py | 31 ++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 test/framework/modules/intel-compilers/2024.0.0 diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py index 1a21b21943..c33b5ee1a2 100644 --- a/easybuild/toolchains/compiler/intel_compilers.py +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -48,7 +48,8 @@ class IntelCompilers(IntelIccIfort): 'oneapi': (None, "Use oneAPI compilers icx/icpx/ifx instead of classic compilers"), 'oneapi_c_cxx': (None, "Use oneAPI C/C++ compilers icx/icpx instead of classic Intel C/C++ compilers " "(auto-enabled for Intel compilers version 2022.2.0, or newer)"), - 'oneapi_fortran': (False, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler"), + 'oneapi_fortran': (None, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler " + "(auto-enabled for Intel compilers version 2024.0.0, or newer)"), }) def _set_compiler_vars(self): @@ -75,6 +76,9 @@ def set_variables(self): # auto-enable use of oneAPI C/C++ compilers for sufficiently recent versions of Intel compilers comp_ver = self.get_software_version(self.COMPILER_MODULE_NAME)[0] if LooseVersion(comp_ver) >= LooseVersion('2022.2.0'): + if LooseVersion(comp_ver) >= LooseVersion('2024.0.0'): + if self.options.get('oneapi_fortran', None) is None: + self.options['oneapi_fortran'] = True if self.options.get('oneapi_c_cxx', None) is None: self.options['oneapi_c_cxx'] = True diff --git a/test/framework/modules/intel-compilers/2024.0.0 b/test/framework/modules/intel-compilers/2024.0.0 new file mode 100644 index 0000000000..a5c0267f9d --- /dev/null +++ b/test/framework/modules/intel-compilers/2024.0.0 @@ -0,0 +1,37 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +Intel C, C++ & Fortran compilers (classic and oneAPI) + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html + } +} + +module-whatis {Description: Intel C, C++ & Fortran compilers (classic and oneAPI)} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html} + +set root /tmp/intel-compilers/2024.0.0 + +conflict intel-compilers + +prepend-path CPATH $root/tbb/2021.11/include +prepend-path LD_LIBRARY_PATH $root/compiler/2024.0/linux/lib +prepend-path LD_LIBRARY_PATH $root/tbb/2021.11/lib/intel64/gcc4.8 +prepend-path LIBRARY_PATH $root/compiler/2024.0/linux/lib +prepend-path LIBRARY_PATH $root/tbb/2021.11/lib/intel64/gcc4.8 +prepend-path MANPATH $root/compiler/2024.0/share/man +prepend-path OCL_ICD_FILENAMES $root/compiler/2024.0/lib/libintelocl.so +prepend-path PATH $root/compiler/2024.0/bin +prepend-path TBBROOT $root/tbb/2021.11 +setenv EBROOTINTELMINCOMPILERS "$root" +setenv EBVERSIONINTELMINCOMPILERS "2024.0.0" +setenv EBDEVELINTELMINCOMPILERS "$root/easybuild/Core-intel-compilers-2024.0.0-easybuild-devel" + +# Built with EasyBuild version 4.8.2 diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0e53639491..dad20a91fc 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1496,6 +1496,37 @@ def test_intel_toolchain_oneapi(self): self.assertEqual(os.getenv('F90'), 'ifx') self.assertEqual(os.getenv('FC'), 'ifx') + self.modtool.purge() + tc = self.get_toolchain('intel-compilers', version='2024.0.0') + tc.prepare() + + # by default (for version >= 2024.0.0): oneAPI C/C++ compiler + oneAPI Fortran compiler + self.assertEqual(os.getenv('CC'), 'icx') + self.assertEqual(os.getenv('CXX'), 'icpx') + self.assertEqual(os.getenv('F77'), 'ifx') + self.assertEqual(os.getenv('F90'), 'ifx') + self.assertEqual(os.getenv('FC'), 'ifx') + + self.modtool.purge() + tc = self.get_toolchain('intel-compilers', version='2024.0.0') + tc.set_options({'oneapi_fortran': False}) + tc.prepare() + self.assertEqual(os.getenv('CC'), 'icx') + self.assertEqual(os.getenv('CXX'), 'icpx') + self.assertEqual(os.getenv('F77'), 'ifort') + self.assertEqual(os.getenv('F90'), 'ifort') + self.assertEqual(os.getenv('FC'), 'ifort') + + self.modtool.purge() + tc = self.get_toolchain('intel-compilers', version='2024.0.0') + tc.set_options({'oneapi_c_cxx': False, 'oneapi_fortran': False}) + tc.prepare() + self.assertEqual(os.getenv('CC'), 'icc') + self.assertEqual(os.getenv('CXX'), 'icpc') + self.assertEqual(os.getenv('F77'), 'ifort') + self.assertEqual(os.getenv('F90'), 'ifort') + self.assertEqual(os.getenv('FC'), 'ifort') + self.modtool.purge() tc = self.get_toolchain('intel', version='2021b') tc.set_options({'oneapi_c_cxx': True}) From 05d66237630f87b8435721a06bfcae805abe1bf0 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 27 Jun 2024 14:46:01 +0000 Subject: [PATCH 354/430] We now have 111 test modules! --- test/framework/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 195bf0339c..8ba4f4ac65 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -55,7 +55,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 110 +TEST_MODULES_COUNT = 111 class ModulesTest(EnhancedTestCase): From 92fb40673017ffe4d19b38af9d22a5f680a14ecb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 28 Jun 2024 14:53:05 +0200 Subject: [PATCH 355/430] fix fetch progress bar showing to many files As we also call `obtain_file` for `checksum.json` the progress bar will show one file to many. E.g. > Fetching files: 100% (4/3) Ignore that file for updating the progressbar --- easybuild/framework/easyblock.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 80cddf8819..f2532df340 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -754,7 +754,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No """ srcpaths = source_paths() - update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename) + # We don't account for the checksums file in the progress bar + if filename != 'checksum.json': + update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename) if alt_location is None: location = self.name From 3e445baa152b5f27b758362a22d8187d28d55f6b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 28 Jun 2024 14:09:49 +0000 Subject: [PATCH 356/430] Resolve internal imkl>=2021 version subdir via "latest" symlink This is already done in the easyblock, and needed since the "latest" symlink pointed to the full version in versions < 2024 but to a shorter version (e.g. 2024.2) in 2024 versions. --- easybuild/toolchains/linalg/intelmkl.py | 3 +++ test/framework/toolchain.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 8a32d684d9..9dc89da827 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -142,6 +142,7 @@ def _set_blas_variables(self): self.variables.nappend_el('CFLAGS', 'DMKL_ILP64') # exact paths/linking statements depend on imkl version + root = self.get_software_root(self.BLAS_MODULE_NAME)[0] found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] ver = LooseVersion(found_version) if ver < LooseVersion('10.3'): @@ -156,6 +157,8 @@ def _set_blas_variables(self): found_version) else: if ver >= LooseVersion('2021'): + if os.path.islink(os.path.join(root, 'mkl', 'latest')): + found_version = os.readlink(os.path.join(root, 'mkl', 'latest')) basedir = os.path.join('mkl', found_version) else: basedir = 'mkl' diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0e53639491..3307426bfd 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1321,8 +1321,11 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): ]) write_file(imkl_fftw_module_path, imkl_fftw_mod_txt) - subdir = 'mkl/%s/lib/intel64' % imklver + # put "latest" symbolic link to short version, used in newer MKL + imklshortver = '.'.join(imklver.split('.')[:2]) + subdir = 'mkl/%s/lib/intel64' % imklshortver os.makedirs(os.path.join(imkl_dir, subdir)) + os.symlink(imklshortver, os.path.join(imkl_dir, 'mkl', 'latest')) for fftlib in mkl_libs: write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo') subdir = 'lib' From a4bf9d206df8b52ef1a988f6d808f29f17ac36e7 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 28 Jun 2024 14:41:43 +0000 Subject: [PATCH 357/430] Adjust internal version to short one in intel FFT test --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 3307426bfd..e77bf6abee 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1201,7 +1201,7 @@ def test_fft_env_vars_intel(self): self.assertEqual(tc.get_variable('LIBFFT'), libfft) self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) - fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4.0/lib/intel64') + fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4/lib/intel64') self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) tc = self.get_toolchain('intel', version='2021b') From eed9e27b24624db200e0c740adf7e551794f0148 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 28 Jun 2024 15:26:50 +0200 Subject: [PATCH 358/430] reuse pre-computed checksums We calculate MD5 and SHA256 checksums as part of the log output which might take considerable time. Instead of recomputing it in `verify_checksum` we can reuse this value. --- easybuild/framework/easyblock.py | 9 +++++++-- easybuild/tools/filetools.py | 12 +++++++++--- test/framework/filetools.py | 19 ++++++++++++++++++- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 80cddf8819..57d28e86ea 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -674,14 +674,16 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): src_fn = os.path.basename(src_path) # report both MD5 and SHA256 checksums, since both are valid default checksum types + src_checksums = {} for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256): src_checksum = compute_checksum(src_path, checksum_type=checksum_type) + src_checksums[checksum_type] = src_checksum self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum) # verify checksum (if provided) self.log.debug('Verifying checksums for extension source...') fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0) - if verify_checksum(src_path, fn_checksum): + if verify_checksum(src_path, fn_checksum, src_checksums): self.log.info('Checksum for extension source %s verified', src_fn) elif build_option('ignore_checksums'): print_warning("Ignoring failing checksum verification for %s" % src_fn) @@ -700,12 +702,15 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): ext_src.update({'patches': ext_patches}) if verify_checksums: + computed_checksums = {} for patch in ext_patches: patch = patch['path'] + computed_checksums[patch] = {} # report both MD5 and SHA256 checksums, # since both are valid default checksum types for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256): checksum = compute_checksum(patch, checksum_type=checksum_type) + computed_checksums[patch][checksum_type] = checksum self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum) # verify checksum (if provided) @@ -715,7 +720,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): patch_fn = os.path.basename(patch) checksum = self.get_checksum_for(checksums, filename=patch_fn, index=idx+1) - if verify_checksum(patch, checksum): + if verify_checksum(patch, checksum, computed_checksums[patch]): self.log.info('Checksum for extension patch %s verified', patch_fn) elif build_option('ignore_checksums'): print_warning("Ignoring failing checksum verification for %s" % patch_fn) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 695043a595..700bb1401e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1242,7 +1242,7 @@ def calc_block_checksum(path, algorithm): return algorithm.hexdigest() -def verify_checksum(path, checksums): +def verify_checksum(path, checksums, computed_checksums=None): """ Verify checksum of specified file. @@ -1303,8 +1303,14 @@ def verify_checksum(path, checksums): "2-tuple (type, value), or tuple of alternative checksum specs.", checksum) - actual_checksum = compute_checksum(path, typ) - _log.debug("Computed %s checksum for %s: %s (correct checksum: %s)" % (typ, path, actual_checksum, checksum)) + if computed_checksums is not None and typ in computed_checksums: + actual_checksum = computed_checksums[typ] + computed_str = 'Precomputed' + else: + actual_checksum = compute_checksum(path, typ) + computed_str = 'Computed' + _log.debug("%s %s checksum for %s: %s (correct checksum: %s)" % + (computed_str, typ, path, actual_checksum, checksum)) if actual_checksum != checksum: return False diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 63cd0a93b7..ae32993d9e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -33,6 +33,7 @@ """ import datetime import glob +import logging import os import re import shutil @@ -297,10 +298,26 @@ def test_checksums(self): 'b7297da8b547d5e74b851d7c4e475900cec4744df0f887ae5c05bf1757c224b4', } + old_log_level = ft._log.getEffectiveLevel() + ft._log.setLevel(logging.DEBUG) # make sure checksums computation/verification is correct for checksum_type, checksum in known_checksums.items(): self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum) - self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum))) + with self.log_to_testlogfile(): + self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum))) + self.assertIn('Computed ' + checksum_type, ft.read_file(self.logfile)) + # Passing precomputed checksums reuses it + with self.log_to_testlogfile(): + computed_checksums = {checksum_type: checksum} + self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum), computed_checksums)) + self.assertIn('Precomputed ' + checksum_type, ft.read_file(self.logfile)) + # If the type isn't contained the checksum will be computed + with self.log_to_testlogfile(): + computed_checksums = {'doesnt exist': 'checksum'} + self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum), computed_checksums)) + self.assertIn('Computed ' + checksum_type, ft.read_file(self.logfile)) + + ft._log.setLevel(old_log_level) # default checksum type is MD5 self.assertEqual(ft.compute_checksum(fp), known_checksums['md5']) From 17983102c5497197e8466c890a487cf962048728 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 3 Jul 2024 13:48:18 +0000 Subject: [PATCH 359/430] gmpflf FlexiBLAS dependency --- easybuild/toolchains/gmpflf.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py index f040a161da..eb754cc873 100644 --- a/easybuild/toolchains/gmpflf.py +++ b/easybuild/toolchains/gmpflf.py @@ -45,3 +45,12 @@ class Gmpflf(Gmpich, OpenBLAS, FlexiBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, MPICH, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'gmpflf' SUBTOOLCHAIN = [Gmpich.NAME, Golf.NAME, Gfbf.NAME] + + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + super(Gmpflf, self).__init__(*args, **kwargs) + constants = ('BLAS_MODULE_NAME', 'BLAS_LIB', 'BLAS_LIB_MT', 'BLAS_FAMILY', + 'LAPACK_MODULE_NAME', 'LAPACK_IS_BLAS', 'LAPACK_FAMILY') + + for constant in constants: + setattr(self, constant, getattr(FlexiBLAS, constant)) From 7ae687c83bf6509431fae93086413fc4bff85df9 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:19:25 +0200 Subject: [PATCH 360/430] Update gmpflf.py --- easybuild/toolchains/gmpflf.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py index eb754cc873..99a01935d4 100644 --- a/easybuild/toolchains/gmpflf.py +++ b/easybuild/toolchains/gmpflf.py @@ -34,23 +34,13 @@ """ from easybuild.toolchains.gmpich import Gmpich from easybuild.toolchains.gfbf import Gfbf -from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.flexiblas import FlexiBLAS from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK -class Gmpflf(Gmpich, OpenBLAS, FlexiBLAS, ScaLAPACK, Fftw): +class Gmpflf(Gmpich, FlexiBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, MPICH, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'gmpflf' - SUBTOOLCHAIN = [Gmpich.NAME, Golf.NAME, Gfbf.NAME] - - def __init__(self, *args, **kwargs): - """Toolchain constructor.""" - super(Gmpflf, self).__init__(*args, **kwargs) - constants = ('BLAS_MODULE_NAME', 'BLAS_LIB', 'BLAS_LIB_MT', 'BLAS_FAMILY', - 'LAPACK_MODULE_NAME', 'LAPACK_IS_BLAS', 'LAPACK_FAMILY') - - for constant in constants: - setattr(self, constant, getattr(FlexiBLAS, constant)) + SUBTOOLCHAIN = [Gmpich.NAME, Gfbf.NAME] From e5d127d4701756b497a89af6864ecd4c227473b1 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:21:18 +0200 Subject: [PATCH 361/430] Update gmpflf.py --- easybuild/toolchains/gmpflf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py index 99a01935d4..f5db523dc1 100644 --- a/easybuild/toolchains/gmpflf.py +++ b/easybuild/toolchains/gmpflf.py @@ -41,6 +41,6 @@ class Gmpflf(Gmpich, FlexiBLAS, ScaLAPACK, Fftw): - """Compiler toolchain with GCC, MPICH, OpenBLAS, ScaLAPACK and FFTW.""" + """Compiler toolchain with GCC, MPICH, FlexiBLAS, ScaLAPACK and FFTW.""" NAME = 'gmpflf' SUBTOOLCHAIN = [Gmpich.NAME, Gfbf.NAME] From 66b82a1805fa223cf6263f89b9c82af30cba2bd5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 3 Jul 2024 17:06:32 +0200 Subject: [PATCH 362/430] Update docstring --- easybuild/tools/filetools.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 700bb1401e..50118c2e28 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1247,7 +1247,11 @@ def verify_checksum(path, checksums, computed_checksums=None): Verify checksum of specified file. :param path: path of file to verify checksum of - :param checksums: checksum values (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b') + :param checksums: checksum values to compare to + (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b') + :param computed_checksums: Optional dictionary of (current) checksum(s) for this file + indexed by the checksum type (e.g. 'sha256'). + Each existing entry will be used, missing ones will be computed. """ filename = os.path.basename(path) From 9846b9a9f08cdb6b90d71a709887358eed1254df Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jul 2024 11:31:41 +0200 Subject: [PATCH 363/430] remove unused import for OpenBLAS in gmpflf toolchain definition --- easybuild/toolchains/gmpflf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py index f5db523dc1..30d10a90f7 100644 --- a/easybuild/toolchains/gmpflf.py +++ b/easybuild/toolchains/gmpflf.py @@ -36,7 +36,6 @@ from easybuild.toolchains.gfbf import Gfbf from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.flexiblas import FlexiBLAS -from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK From e4bf7743a0bfc213e237646d64373cffc73f6070 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 4 Jul 2024 11:57:25 +0200 Subject: [PATCH 364/430] CI: Allow using Node 16 actions Required after https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default --- .github/workflows/end2end.yml | 1 + .github/workflows/unit_tests_python2.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7327d876ba..e74b65cc79 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -18,6 +18,7 @@ jobs: fail-fast: false container: image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 + env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions steps: - name: Check out the repo uses: actions/checkout@v3 diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 1b921ee83c..aa00f9a2fd 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -16,6 +16,7 @@ jobs: # CentOS 7.9 container that already includes Lmod & co, # see https://github.com/easybuilders/easybuild-containers image: ghcr.io/easybuilders/centos-7.9-amd64 + env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions steps: - uses: actions/checkout@v3 From 669806bc89951f02410d6faaf35c1f69576e4b3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Jul 2024 15:30:42 +0200 Subject: [PATCH 365/430] comit -> commit --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 68ddba632a..df10ec859e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1590,7 +1590,7 @@ def check_included_multiple(included_easyblocks_from, source): check_included_multiple(included_from_commit, "commit %s" % easyblock_commit) for easyblock in included_from_commit: - print_msg("easyblock %s included from comit %s" % (easyblock, easyblock_commit), log=log) + print_msg("easyblock %s included from commit %s" % (easyblock, easyblock_commit), log=log) include_easyblocks(options.tmpdir, easyblocks_from_commit) From 2874f8f041b521bb2edbf6490800cb2d52927cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Jul 2024 15:30:46 +0200 Subject: [PATCH 366/430] comit -> commit --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 94c4372f91..a0ebac4324 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2225,7 +2225,7 @@ def test_xxx_include_easyblocks_from_commit(self): import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) - pattern = "== easyblock binary.py included from comit %s" % test_commit + pattern = "== easyblock binary.py included from commit %s" % test_commit self.assertEqual(stderr, '') self.assertIn(pattern, stdout) From 66ec1af623c4e7c0a9140209369ad4911cfc2d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Jul 2024 14:59:41 +0200 Subject: [PATCH 367/430] don't use special flags for strict and precise toolchain options on RISC-V --- easybuild/toolchains/compiler/gcc.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 548ac41187..bbf4027e59 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -78,6 +78,13 @@ class Gcc(Compiler): COMPILER_UNIQUE_OPTION_MAP['strict'] = no_recip_alternative COMPILER_UNIQUE_OPTION_MAP['precise'] = no_recip_alternative + # gcc on RISC-V does not support -mno-recip, -mieee-fp, -mfno-math-errno... + # https://gcc.gnu.org/onlinedocs/gcc/RISC-V-Options.html + # there are no good alternatives, so stick to the default flags + if systemtools.get_cpu_family() == systemtools.RISCV: + COMPILER_UNIQUE_OPTION_MAP['strict'] = [] + COMPILER_UNIQUE_OPTION_MAP['precise'] = [] + # used when 'optarch' toolchain option is enabled (and --optarch is not specified) COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { (systemtools.AARCH32, systemtools.ARM): 'mcpu=native', # implies -march=native and -mtune=native From 37858f110a69ccb3dbae39bd171d81895434dc4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Jul 2024 11:27:32 +0200 Subject: [PATCH 368/430] also remove unsupported flags for loose and veryloose on RISC-V --- easybuild/toolchains/compiler/gcc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index bbf4027e59..e9748f3bda 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -84,6 +84,8 @@ class Gcc(Compiler): if systemtools.get_cpu_family() == systemtools.RISCV: COMPILER_UNIQUE_OPTION_MAP['strict'] = [] COMPILER_UNIQUE_OPTION_MAP['precise'] = [] + COMPILER_UNIQUE_OPTION_MAP['loose'] = ['fno-math-errno'] + COMPILER_UNIQUE_OPTION_MAP['verloose'] = ['fno-math-errno'] # used when 'optarch' toolchain option is enabled (and --optarch is not specified) COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { From 13eb8b602586d07a261d7fa098831030acd0b00f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 22 Jul 2024 16:34:12 +0200 Subject: [PATCH 369/430] Add cuda_cc_space_sep variant that does not have periods, e.g. '80 90' if cuda compute capabilities is 8.0,9.0 --- easybuild/framework/easyconfig/templates.py | 3 +++ test/framework/easyconfig.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 9da6e9a2b9..eae4284687 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -97,6 +97,8 @@ "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), + ('cuda_cc_space_sep_no_period', + "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90')."), ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), @@ -367,6 +369,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) if cuda_compute_capabilities: template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities) template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) + template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_compute_capabilities) template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities) sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 4b63dc605b..c873b9aa16 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4606,7 +4606,7 @@ def test_cuda_compute_capabilities(self): toolchain = SYSTEM cuda_compute_capabilities = ['5.1', '7.0', '7.1'] installopts = '%(cuda_compute_capabilities)s' - preinstallopts = '%(cuda_cc_space_sep)s' + preinstallopts = 'period="%(cuda_cc_space_sep)s" noperiod="%(cuda_cc_space_sep_no_period)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' @@ -4615,7 +4615,7 @@ def test_cuda_compute_capabilities(self): ec = EasyConfig(self.eb_file) self.assertEqual(ec['installopts'], '5.1,7.0,7.1') - self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') + self.assertEqual(ec['preinstallopts'], 'period="5.1 7.0 7.1" noperiod="51 70 71"') self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' 'space="sm_51 sm_70 sm_71"') From 49455ae88801d9760ba43965a12a6e2ba310c25a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 22 Jul 2024 16:38:24 +0200 Subject: [PATCH 370/430] Shorten local variable name --- easybuild/framework/easyconfig/templates.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index eae4284687..c6cfe747aa 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -93,8 +93,8 @@ ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" "as specify by the --sysroot configuration option"), ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), - ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " - "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), + ('cuda_cc', "Comma-separated list of CUDA compute capabilities, as specified via " + "--cuda-compute-capabilities configuration option or via cuda_cc easyconfig parameter"), ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), ('cuda_cc_space_sep_no_period', @@ -365,14 +365,14 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # step 6. CUDA compute capabilities # Use the commandline / easybuild config option if given, else use the value from the EC (as a default) - cuda_compute_capabilities = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities') - if cuda_compute_capabilities: - template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities) - template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) - template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_compute_capabilities) - template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) - template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities) - sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] + cuda_cc = build_option('cuda_cc') or config.get('cuda_cc') + if cuda_cc: + template_values['cuda_compute_capabilities'] = ','.join(cuda_cc) + template_values['cuda_cc_space_sep'] = ' '.join(cuda_cc) + template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_cc) + template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_cc) + template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_cc) + sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_cc] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) From f17edd1f763074ad641f183e2f973093d534f0ef Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 22 Jul 2024 16:40:35 +0200 Subject: [PATCH 371/430] Revert shortening when it concerns the real build option cuda_compute_capabilities, and not the local variable --- easybuild/framework/easyconfig/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index c6cfe747aa..df46e5eaeb 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -365,7 +365,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # step 6. CUDA compute capabilities # Use the commandline / easybuild config option if given, else use the value from the EC (as a default) - cuda_cc = build_option('cuda_cc') or config.get('cuda_cc') + cuda_cc = build_option('cuda_cc') or config.get('cuda_compute_capabilities') if cuda_cc: template_values['cuda_compute_capabilities'] = ','.join(cuda_cc) template_values['cuda_cc_space_sep'] = ' '.join(cuda_cc) From 37a473deebb7ff9a91c103ca417793041769fd40 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 22 Jul 2024 17:14:59 +0200 Subject: [PATCH 372/430] Revert shortening when it's really about the cuda compute capabilities build option, instead of the local var --- easybuild/framework/easyconfig/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index df46e5eaeb..88174c8393 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -365,7 +365,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # step 6. CUDA compute capabilities # Use the commandline / easybuild config option if given, else use the value from the EC (as a default) - cuda_cc = build_option('cuda_cc') or config.get('cuda_compute_capabilities') + cuda_cc = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities') if cuda_cc: template_values['cuda_compute_capabilities'] = ','.join(cuda_cc) template_values['cuda_cc_space_sep'] = ' '.join(cuda_cc) From c6a572399e22bcafd7f050c081af9bc2af60d3d1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 22 Jul 2024 17:36:12 +0200 Subject: [PATCH 373/430] Fix name of cuda_compute_capabilities template, it was accidentally replaced --- easybuild/framework/easyconfig/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 88174c8393..e1ac187eee 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -93,7 +93,7 @@ ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" "as specify by the --sysroot configuration option"), ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), - ('cuda_cc', "Comma-separated list of CUDA compute capabilities, as specified via " + ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " "--cuda-compute-capabilities configuration option or via cuda_cc easyconfig parameter"), ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), From 1674f1eb5acd34f645266b82be0860bf547cd445 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 22 Jul 2024 23:38:42 +0200 Subject: [PATCH 374/430] Forgot to change second expected pattern in unit test. Did so now --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c873b9aa16..eb21430f47 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4625,7 +4625,7 @@ def test_cuda_compute_capabilities(self): init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']}) ec = EasyConfig(self.eb_file) self.assertEqual(ec['installopts'], '4.2,6.3') - self.assertEqual(ec['preinstallopts'], '4.2 6.3') + self.assertEqual(ec['preinstallopts'], 'period="4.2 6.3" noperiod="42 63"') self.assertEqual(ec['prebuildopts'], '4.2;6.3') self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' 'space="sm_42 sm_63"') From 0da94f63895efd9c4978d1966280e304f031e2d9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 23 Jul 2024 15:51:11 +0200 Subject: [PATCH 375/430] `toolchainopts` are not supported for `SYSTEM` compiler --- easybuild/toolchains/system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/toolchains/system.py b/easybuild/toolchains/system.py index 8a8b17cfaa..1a6a79c34f 100644 --- a/easybuild/toolchains/system.py +++ b/easybuild/toolchains/system.py @@ -37,3 +37,5 @@ class SystemToolchain(SystemCompiler): """System toolchain.""" NAME = SYSTEM_TOOLCHAIN_NAME + COMPILER_UNIQUE_OPTS = None + COMPILER_SHARED_OPTS = None From c27e92be5576b92c3c0daa601a300217b5505a6e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 23 Jul 2024 17:03:39 +0200 Subject: [PATCH 376/430] Option mappings also need to be reset --- easybuild/toolchains/system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/toolchains/system.py b/easybuild/toolchains/system.py index 1a6a79c34f..db806a2a50 100644 --- a/easybuild/toolchains/system.py +++ b/easybuild/toolchains/system.py @@ -39,3 +39,5 @@ class SystemToolchain(SystemCompiler): NAME = SYSTEM_TOOLCHAIN_NAME COMPILER_UNIQUE_OPTS = None COMPILER_SHARED_OPTS = None + COMPILER_UNIQUE_OPTION_MAP = None + COMPILER_SHARED_OPTION_MAP = None From 16ce1ed1f86dcf6a560e413da630978aa58699c7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 24 Jul 2024 13:57:57 +0200 Subject: [PATCH 377/430] Move the changes to the parent class --- easybuild/toolchains/compiler/systemcompiler.py | 7 +++++++ easybuild/toolchains/system.py | 4 ---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/easybuild/toolchains/compiler/systemcompiler.py b/easybuild/toolchains/compiler/systemcompiler.py index d1fd06d4d9..140ae3b159 100644 --- a/easybuild/toolchains/compiler/systemcompiler.py +++ b/easybuild/toolchains/compiler/systemcompiler.py @@ -40,3 +40,10 @@ class SystemCompiler(Compiler): """System compiler""" COMPILER_MODULE_NAME = [] COMPILER_FAMILY = TC_CONSTANT_SYSTEM + + # The system compiler does not currently support even the shared options + # (changing this would require updating set_minimal_build_env() of the toolchain class) + COMPILER_UNIQUE_OPTS = None + COMPILER_SHARED_OPTS = None + COMPILER_UNIQUE_OPTION_MAP = None + COMPILER_SHARED_OPTION_MAP = None diff --git a/easybuild/toolchains/system.py b/easybuild/toolchains/system.py index db806a2a50..8a8b17cfaa 100644 --- a/easybuild/toolchains/system.py +++ b/easybuild/toolchains/system.py @@ -37,7 +37,3 @@ class SystemToolchain(SystemCompiler): """System toolchain.""" NAME = SYSTEM_TOOLCHAIN_NAME - COMPILER_UNIQUE_OPTS = None - COMPILER_SHARED_OPTS = None - COMPILER_UNIQUE_OPTION_MAP = None - COMPILER_SHARED_OPTION_MAP = None From fba1825ebd23a9a0520d02ecf3e8ec060df046af Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 24 Jul 2024 16:52:47 +0200 Subject: [PATCH 378/430] Make unrelated warning disappear --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index c9f42891ec..5fa01efb5f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -191,7 +191,7 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test|CryptographyDeprecationWarning: TripleDES has been moved" # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) From 262ff858c4c5388f025bcfc9d919493144d3777d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 24 Jul 2024 17:16:07 +0200 Subject: [PATCH 379/430] Make unrelated warning from CI disappear --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5fa01efb5f..b6b79ac4d5 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -191,7 +191,7 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test|CryptographyDeprecationWarning: TripleDES has been moved" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test|CryptographyDeprecationWarning: TripleDES has been moved|algorithms.TripleDES" # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) From 72f41065e41b3825c09e15da915267112abcea6d Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 30 Jul 2024 10:06:47 +0200 Subject: [PATCH 380/430] Use a test easyconfig where toolchainopts are allowed --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 2978e64b1a..eeff304c16 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2820,7 +2820,7 @@ def grab_gcc_rpath_wrapper_args(): # test use of rpath toolchain option test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')) + toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0-gompi-2018a.eb')) toy_ec_txt += "\ntoolchainopts = {'rpath': False}\n" toy_ec = os.path.join(self.test_prefix, 'toy.eb') write_file(toy_ec, toy_ec_txt) From 0444bd2fdb66c6e909c58d1382527c4f8e018bf6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 30 Jul 2024 14:33:17 +0200 Subject: [PATCH 381/430] Don't use SYSTEM toolchain for toolchain tests --- test/framework/toolchain.py | 2 +- test/framework/toy_build.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 2ca15d26d6..c140c8d042 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -91,7 +91,7 @@ def get_toolchain(self, name, version=None): def test_toolchain(self): """Test whether toolchain is initialized correctly.""" test_ecs = os.path.join('test', 'framework', 'easyconfigs', 'test_ecs') - ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4.eb')) + ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4-GCC-4.9.3-2.26.eb')) ec = EasyConfig(ec_file, validate=False) tc = ec.toolchain self.assertIn('debug', tc.options) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index eeff304c16..11f4516977 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2821,7 +2821,7 @@ def grab_gcc_rpath_wrapper_args(): # test use of rpath toolchain option test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0-gompi-2018a.eb')) - toy_ec_txt += "\ntoolchainopts = {'rpath': False}\n" + toy_ec_txt += "\ntoolchainopts = {'rpath': False}\n" # overwrites existing toolchainopts toy_ec = os.path.join(self.test_prefix, 'toy.eb') write_file(toy_ec, toy_ec_txt) with self.mocked_stdout_stderr(): From 27a869b0b432e3a0ea050453fbc351ec859430b1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 24 Jul 2024 17:16:07 +0200 Subject: [PATCH 382/430] Make unrelated warning from CI disappear --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 223d1da646..1072bf421d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -187,7 +187,7 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test|CryptographyDeprecationWarning: TripleDES has been moved|algorithms.TripleDES" # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) From 5724b91ee1209264a959ca32faf70535883a28f1 Mon Sep 17 00:00:00 2001 From: Patrice Peterson Date: Thu, 1 Aug 2024 14:57:07 +0200 Subject: [PATCH 383/430] Use XDG_CONFIG_DIRS value from XDG basedir spec The spec mandates `/etc/xdg` instead of `/etc` for the default value of `XDG_CONFIG_DIRS` [1]. [1] https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables --- easybuild/tools/options.py | 4 ++-- test/framework/options.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 128ff8561b..935f450782 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -123,7 +123,7 @@ def terminal_supports_colors(stream): CONFIG_ENV_VAR_PREFIX = 'EASYBUILD' XDG_CONFIG_HOME = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), ".config")) -XDG_CONFIG_DIRS = os.environ.get('XDG_CONFIG_DIRS', '/etc').split(os.pathsep) +XDG_CONFIG_DIRS = os.environ.get('XDG_CONFIG_DIRS', '/etc/xdg').split(os.pathsep) DEFAULT_SYS_CFGFILES = [f for d in XDG_CONFIG_DIRS for f in sorted(glob.glob(os.path.join(d, 'easybuild.d', '*.cfg')))] DEFAULT_USER_CFGFILE = os.path.join(XDG_CONFIG_HOME, 'easybuild', 'config.cfg') @@ -1313,7 +1313,7 @@ def show_default_configfiles(self): '', "* user-level: %s" % os.path.join('${XDG_CONFIG_HOME:-$HOME/.config}', 'easybuild', 'config.cfg'), " -> %s => %s" % (DEFAULT_USER_CFGFILE, ('not found', 'found')[os.path.exists(DEFAULT_USER_CFGFILE)]), - "* system-level: %s" % os.path.join('${XDG_CONFIG_DIRS:-/etc}', 'easybuild.d', '*.cfg'), + "* system-level: %s" % os.path.join('${XDG_CONFIG_DIRS:-/etc/xdg}', 'easybuild.d', '*.cfg'), " -> %s => %s" % (system_cfg_glob_paths, ', '.join(DEFAULT_SYS_CFGFILES) or "(no matches)"), '', "Default list of existing configuration files (%d): %s" % (found_cfgfile_cnt, found_cfgfile_list), diff --git a/test/framework/options.py b/test/framework/options.py index c9c92663e6..aba8752f69 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3493,7 +3493,7 @@ def test_show_default_configfiles(self): '', "* user-level: ${XDG_CONFIG_HOME:-$HOME/.config}/easybuild/config.cfg", " -> %s", - "* system-level: ${XDG_CONFIG_DIRS:-/etc}/easybuild.d/*.cfg", + "* system-level: ${XDG_CONFIG_DIRS:-/etc/xdg}/easybuild.d/*.cfg", " -> %s/easybuild.d/*.cfg => ", ]) @@ -3508,12 +3508,12 @@ def test_show_default_configfiles(self): homecfgfile_str += " => found" else: homecfgfile_str += " => not found" - expected = expected_tmpl % ('(not set)', '(not set)', homecfgfile_str, '{/etc}') + expected = expected_tmpl % ('(not set)', '(not set)', homecfgfile_str, '{/etc/xdg}') self.assertIn(expected, logtxt) # to predict the full output, we need to take control over $HOME and $XDG_CONFIG_DIRS os.environ['HOME'] = self.test_prefix - xdg_config_dirs = os.path.join(self.test_prefix, 'etc') + xdg_config_dirs = os.path.join(self.test_prefix, 'etc', 'xdg') os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs expected_tmpl += '\n'.join([ @@ -3538,12 +3538,12 @@ def test_show_default_configfiles(self): xdg_config_home = os.path.join(self.test_prefix, 'home') os.environ['XDG_CONFIG_HOME'] = xdg_config_home - xdg_config_dirs = [os.path.join(self.test_prefix, 'etc'), os.path.join(self.test_prefix, 'moaretc')] + xdg_config_dirs = [os.path.join(self.test_prefix, 'etc', 'xdg'), os.path.join(self.test_prefix, 'moaretc')] os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join(xdg_config_dirs) # put various dummy cfgfiles in place cfgfiles = [ - os.path.join(self.test_prefix, 'etc', 'easybuild.d', 'config.cfg'), + os.path.join(self.test_prefix, 'etc', 'xdg', 'easybuild.d', 'config.cfg'), os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'bar.cfg'), os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'foo.cfg'), os.path.join(xdg_config_home, 'easybuild', 'config.cfg'), From 0abc676f13af0ba6bc1c859186b95e0e6858e45e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 5 Aug 2024 10:44:46 +0200 Subject: [PATCH 384/430] Document and fix use of ignore parameter of template_constant_dict --- easybuild/framework/easyconfig/templates.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 0e2958ede7..a781caca19 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -277,10 +277,9 @@ def template_constant_dict(config, ignore=None, toolchain=None): """Create a dict for templating the values in the easyconfigs. - - config is a dict with the structure of EasyConfig._config + - config -- Dict with the structure of EasyConfig._config + - ignore -- List of template names to ignore """ - # TODO find better name - # ignore if ignore is None: ignore = [] # make dict @@ -296,7 +295,7 @@ def template_constant_dict(config, ignore=None, toolchain=None): # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: - if name in ignore: + if name[0] in ignore: continue # check if this template name is already handled From f5be9f59f56d7ca1b9156ee8a90da49a33ea063a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 5 Aug 2024 12:31:56 +0200 Subject: [PATCH 385/430] Remove duplicate entry from ALTERNATIVE_EASYCONFIG_TEMPLATES --- easybuild/framework/easyconfig/templates.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index a781caca19..92bf81fdcf 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -169,11 +169,8 @@ 'build_dir': 'builddir', 'cuda_cc_comma_sep': 'cuda_compute_capabilities', 'cuda_maj_ver': 'cudamajver', - 'cuda_maj_ver': 'cudamajver', - 'cuda_short_ver': 'cudashortver', 'cuda_short_ver': 'cudashortver', 'cuda_ver': 'cudaver', - 'cuda_ver': 'cudaver', 'install_dir': 'installdir', 'java_maj_ver': 'javamajver', 'java_short_ver': 'javashortver', From 0add942886ce4c587d19a41b95eaa8476069178f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 5 Aug 2024 12:33:10 +0200 Subject: [PATCH 386/430] Convert template constants to dicts. Those are easier to handle as we can a) Access the names directly by iterating over the dict(-keys) b) Avoid duplications c) Make the semantic clearer d) Allow direct access of values by template name --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/easyconfig/easyconfig.py | 4 +- .../easyconfig/format/pyheaderconfigobj.py | 4 +- easybuild/framework/easyconfig/templates.py | 373 +++++++++--------- easybuild/framework/extension.py | 2 +- easybuild/tools/docs.py | 81 ++-- test/framework/easyconfig.py | 2 +- 7 files changed, 236 insertions(+), 232 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 123e4f7695..b781bf2218 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3921,7 +3921,7 @@ def update_config_template_run_step(self): """Update the the easyconfig template dictionary with easyconfig.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP names""" for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - self.cfg.template_values[name[0]] = str(getattr(self, name[0], None)) + self.cfg.template_values[name] = str(getattr(self, name, None)) self.cfg.generate_template_values() def skip_step(self, step, skippable): diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 772057fa6b..06d3649a73 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1217,7 +1217,7 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals default_values.update({key: value[0] for key, value in self.extra_options.items()}) self.generate_template_values() - templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS} + templ_const = {quote_py_str(value): name for name, (value, _) in TEMPLATE_CONSTANTS.items()} # create reverse map of templates, to inject template values where possible # longer template values are considered first, shorter template keys get preference over longer ones @@ -1842,7 +1842,7 @@ def get_cuda_cc_template_value(self, key): Returns user-friendly error message in case neither are defined, or if an unknown key is used. """ - if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC): + if key.startswith('cuda_') and any(x == key for x in TEMPLATE_NAMES_DYNAMIC): try: return self.template_values[key] except KeyError: diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 48e6516e8c..c52a459322 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -52,8 +52,8 @@ def build_easyconfig_constants_dict(): """Make a dictionary with all constants that can be used""" all_consts = [ - ('TEMPLATE_CONSTANTS', {x[0]: x[1] for x in TEMPLATE_CONSTANTS}), - ('EASYCONFIG_CONSTANTS', {key: val[0] for key, val in EASYCONFIG_CONSTANTS.items()}), + ('TEMPLATE_CONSTANTS', {name: value for name, (value, _) in TEMPLATE_CONSTANTS.items()}), + ('EASYCONFIG_CONSTANTS', {name: value for name, (value, _) in EASYCONFIG_CONSTANTS.items()}), ('EASYCONFIG_LICENSES', {klass().name: name for name, klass in EASYCONFIG_LICENSES_DICT.items()}), ] err = [] diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 92bf81fdcf..f6df89e682 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -45,15 +45,15 @@ _log = fancylogger.getLogger('easyconfig.templates', fname=False) # derived from easyconfig, but not from ._config directly -TEMPLATE_NAMES_EASYCONFIG = [ - ('module_name', "Module name"), - ('nameletter', "First letter of software name"), - ('toolchain_name', "Toolchain name"), - ('toolchain_version', "Toolchain version"), - ('version_major_minor', "Major.Minor version"), - ('version_major', "Major version"), - ('version_minor', "Minor version"), -] +TEMPLATE_NAMES_EASYCONFIG = { + 'module_name': 'Module name', + 'nameletter': 'First letter of software name', + 'toolchain_name': 'Toolchain name', + 'toolchain_version': 'Toolchain version', + 'version_major_minor': "Major.Minor version", + 'version_major': 'Major version', + 'version_minor': 'Minor version', +} # derived from EasyConfig._config TEMPLATE_NAMES_CONFIG = [ 'bitbucket_account', @@ -71,97 +71,103 @@ 'nameletter', ] # values taken from the EasyBlock before each step -TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = [ - ('builddir', "Build directory"), - ('installdir', "Installation directory"), - ('start_dir', "Directory in which the build process begins"), -] +TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = { + 'builddir': 'Build directory', + 'installdir': 'Installation directory', + 'start_dir': 'Directory in which the build process begins', +} # software names for which to define ver, majver and shortver templates -TEMPLATE_SOFTWARE_VERSIONS = [ - # software name, prefix for *ver, *majver and *shortver - ('CUDA', 'cuda'), - ('CUDAcore', 'cuda'), - ('Java', 'java'), - ('Perl', 'perl'), - ('Python', 'py'), - ('R', 'r'), -] +TEMPLATE_SOFTWARE_VERSIONS = { + # software name -> prefix for *ver, *majver and *shortver + 'CUDA': 'cuda', + 'CUDAcore': 'cuda', + 'Java': 'java', + 'Perl': 'perl', + 'Python': 'py', + 'R': 'r', +} # template values which are only generated dynamically -TEMPLATE_NAMES_DYNAMIC = [ - ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), - ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" - "as specify by the --sysroot configuration option"), - ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), - ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " - "--cuda-compute-capabilities configuration option or via cuda_cc easyconfig parameter"), - ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), - ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), - ('cuda_cc_space_sep_no_period', - "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90')."), - ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), - ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"), - ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"), - ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"), - ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), - ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), -] +TEMPLATE_NAMES_DYNAMIC = { + 'arch': 'System architecture (e.g. x86_64, aarch64, ppc64le, ...)', + 'sysroot': "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" + "as specify by the --sysroot configuration option", + 'mpi_cmd_prefix': 'Prefix command for running MPI programs (with default number of ranks)', + 'cuda_compute_capabilities': "Comma-separated list of CUDA compute capabilities, as specified via " + "--cuda-compute-capabilities configuration option or " + "via cuda_compute_capabilities easyconfig parameter", + 'cuda_cc_cmake': 'List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+', + 'cuda_cc_space_sep': 'Space-separated list of CUDA compute capabilities', + 'cuda_cc_space_sep_no_period': + "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90').", + 'cuda_cc_semicolon_sep': 'Semicolon-separated list of CUDA compute capabilities', + 'cuda_int_comma_sep': 'Comma-separated list of integer CUDA compute capabilities', + 'cuda_int_space_sep': 'Space-separated list of integer CUDA compute capabilities', + 'cuda_int_semicolon_sep': 'Semicolon-separated list of integer CUDA compute capabilities', + 'cuda_sm_comma_sep': 'Comma-separated list of sm_* values that correspond with CUDA compute capabilities', + 'cuda_sm_space_sep': 'Space-separated list of sm_* values that correspond with CUDA compute capabilities', +} # constant templates that can be used in easyconfigs -TEMPLATE_CONSTANTS = [ +# Entry: constant -> (value, doc) +TEMPLATE_CONSTANTS = { # source url constants - ('APACHE_SOURCE', 'https://archive.apache.org/dist/%(namelower)s', - 'apache.org source url'), - ('BITBUCKET_SOURCE', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get', - 'bitbucket.org source url (namelower is used if bitbucket_account easyconfig parameter is not specified)'), - ('BITBUCKET_DOWNLOADS', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads', - 'bitbucket.org downloads url (namelower is used if bitbucket_account easyconfig parameter is not specified)'), - ('CRAN_SOURCE', 'https://cran.r-project.org/src/contrib', - 'CRAN (contrib) source url'), - ('FTPGNOME_SOURCE', 'https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s', - 'http download for gnome ftp server'), - ('GITHUB_SOURCE', 'https://github.com/%(github_account)s/%(name)s/archive', - 'GitHub source URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'), - ('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive', - 'GitHub source URL with lowercase name (if github_account easyconfig ' - 'parameter is not specified, namelower is used in its place)'), - ('GITHUB_RELEASE', 'https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s', - 'GitHub release URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'), - ('GITHUB_LOWER_RELEASE', 'https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s', - 'GitHub release URL with lowercase name (if github_account easyconfig ' - 'parameter is not specified, namelower is used in its place)'), - ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s', - 'download.savannah.gnu.org source url'), - ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s', - 'gnu.org source url'), - ('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files', - 'googlecode.com source url'), - ('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/', - 'launchpad.net source url'), - ('PYPI_SOURCE', 'https://pypi.python.org/packages/source/%(nameletter)s/%(name)s', - 'pypi source url'), # e.g., Cython, Sphinx - ('PYPI_LOWER_SOURCE', 'https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s', - 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ - ('R_SOURCE', 'https://cran.r-project.org/src/base/R-%(version_major)s', - 'cran.r-project.org (base) source url'), - ('SOURCEFORGE_SOURCE', 'https://download.sourceforge.net/%(namelower)s', - 'sourceforge.net source url'), - ('XORG_DATA_SOURCE', 'https://xorg.freedesktop.org/archive/individual/data/', - 'xorg data source url'), - ('XORG_LIB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/lib/', - 'xorg lib source url'), - ('XORG_PROTO_SOURCE', 'https://xorg.freedesktop.org/archive/individual/proto/', - 'xorg proto source url'), - ('XORG_UTIL_SOURCE', 'https://xorg.freedesktop.org/archive/individual/util/', - 'xorg util source url'), - ('XORG_XCB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/xcb/', - 'xorg xcb source url'), + 'APACHE_SOURCE': ('https://archive.apache.org/dist/%(namelower)s', + 'apache.org source url'), + 'BITBUCKET_SOURCE': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get', + 'bitbucket.org source url ' + '(namelower is used if bitbucket_account easyconfig parameter is not specified)'), + 'BITBUCKET_DOWNLOADS': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads', + 'bitbucket.org downloads url ' + '(namelower is used if bitbucket_account easyconfig parameter is not specified)'), + 'CRAN_SOURCE': ('https://cran.r-project.org/src/contrib', + 'CRAN (contrib) source url'), + 'FTPGNOME_SOURCE': ('https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s', + 'http download for gnome ftp server'), + 'GITHUB_SOURCE': ('https://github.com/%(github_account)s/%(name)s/archive', + 'GitHub source URL ' + '(namelower is used if github_account easyconfig parameter is not specified)'), + 'GITHUB_LOWER_SOURCE': ('https://github.com/%(github_account)s/%(namelower)s/archive', + 'GitHub source URL with lowercase name ' + '(namelower is used if github_account easyconfig parameter is not specified)'), + 'GITHUB_RELEASE': ('https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s', + 'GitHub release URL ' + '(namelower is use if github_account easyconfig parameter is not specified)'), + 'GITHUB_LOWER_RELEASE': ('https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s', + 'GitHub release URL with lowercase name (if github_account easyconfig ' + 'parameter is not specified, namelower is used in its place)'), + 'GNU_SAVANNAH_SOURCE': ('https://download-mirror.savannah.gnu.org/releases/%(namelower)s', + 'download.savannah.gnu.org source url'), + 'GNU_SOURCE': ('https://ftpmirror.gnu.org/gnu/%(namelower)s', + 'gnu.org source url'), + 'GOOGLECODE_SOURCE': ('http://%(namelower)s.googlecode.com/files', + 'googlecode.com source url'), + 'LAUNCHPAD_SOURCE': ('https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/', + 'launchpad.net source url'), + 'PYPI_SOURCE': ('https://pypi.python.org/packages/source/%(nameletter)s/%(name)s', + 'pypi source url'), # e.g., Cython, Sphinx + 'PYPI_LOWER_SOURCE': ('https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s', + 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ + 'R_SOURCE': ('https://cran.r-project.org/src/base/R-%(version_major)s', + 'cran.r-project.org (base) source url'), + 'SOURCEFORGE_SOURCE': ('https://download.sourceforge.net/%(namelower)s', + 'sourceforge.net source url'), + 'XORG_DATA_SOURCE': ('https://xorg.freedesktop.org/archive/individual/data/', + 'xorg data source url'), + 'XORG_LIB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/lib/', + 'xorg lib source url'), + 'XORG_PROTO_SOURCE': ('https://xorg.freedesktop.org/archive/individual/proto/', + 'xorg proto source url'), + 'XORG_UTIL_SOURCE': ('https://xorg.freedesktop.org/archive/individual/util/', + 'xorg util source url'), + 'XORG_XCB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/xcb/', + 'xorg xcb source url'), # TODO, not urgent, yet nice to have: # CPAN_SOURCE GNOME KDE_I18N XCONTRIB DEBIAN KDE GENTOO TEX_CTAN MOZILLA_ALL # other constants - ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'), -] + 'SHLIB_EXT': (get_shared_lib_ext(), 'extension for shared libraries'), +} # alternative templates, and their equivalents ALTERNATIVE_EASYCONFIG_TEMPLATES = { @@ -247,13 +253,13 @@ # : (, ), } -extensions = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z'] -for ext in extensions: +EXTENSIONS = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z'] +for ext in EXTENSIONS: suffix = ext.replace('.', '_').upper() - TEMPLATE_CONSTANTS += [ - ('SOURCE_%s' % suffix, '%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext), - ('SOURCELOWER_%s' % suffix, '%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext), - ] + TEMPLATE_CONSTANTS.update({ + 'SOURCE_%s' % suffix: ('%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext), + 'SOURCELOWER_%s' % suffix: ('%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext), + }) for pyver in ('py2.py3', 'py2', 'py3'): if pyver == 'py2.py3': desc = 'Python 2 & Python 3' @@ -261,12 +267,12 @@ else: desc = 'Python ' + pyver[-1] name_infix = pyver.upper() + '_' - TEMPLATE_CONSTANTS += [ - ('SOURCE_%sWHL' % name_infix, '%%(name)s-%%(version)s-%s-none-any.whl' % pyver, - 'Generic (non-compiled) %s wheel package' % desc), - ('SOURCELOWER_%sWHL' % name_infix, '%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver, - 'Generic (non-compiled) %s wheel package with lowercase name' % desc), - ] + TEMPLATE_CONSTANTS.update({ + 'SOURCE_%sWHL' % name_infix: ('%%(name)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package' % desc), + 'SOURCELOWER_%sWHL' % name_infix: ('%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package with lowercase name' % desc), + }) # TODO derived config templates # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) ) @@ -292,14 +298,14 @@ def template_constant_dict(config, ignore=None, toolchain=None): # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: - if name[0] in ignore: + if name in ignore: continue # check if this template name is already handled - if template_values.get(name[0]) is not None: + if template_values.get(name) is not None: continue - if name[0].startswith('toolchain_'): + if name.startswith('toolchain_'): tc = config.get('toolchain') if tc is not None: template_values['toolchain_name'] = tc.get('name', None) @@ -307,7 +313,7 @@ def template_constant_dict(config, ignore=None, toolchain=None): # only go through this once ignore.extend(['toolchain_name', 'toolchain_version']) - elif name[0].startswith('version_'): + elif name.startswith('version_'): # parse major and minor version numbers version = config['version'] if version is not None: @@ -326,87 +332,85 @@ def template_constant_dict(config, ignore=None, toolchain=None): # only go through this once ignore.extend(['version_major', 'version_minor', 'version_major_minor']) - elif name[0].endswith('letter'): + elif name.endswith('letter'): # parse first letters - if name[0].startswith('name'): + if name.startswith('name'): softname = config['name'] if softname is not None: template_values['nameletter'] = softname[0] - elif name[0] == 'module_name': + elif name == 'module_name': template_values['module_name'] = getattr(config, 'short_mod_name', None) else: raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) # step 2: define *ver and *shortver templates - if TEMPLATE_SOFTWARE_VERSIONS: - - name_to_prefix = {name.lower(): pref for name, pref in TEMPLATE_SOFTWARE_VERSIONS} - deps = config.get('dependencies', []) - - # also consider build dependencies for *ver and *shortver templates; - # we need to be a bit careful here, because for iterative installations - # (when multi_deps is used for example) the builddependencies value may be a list of lists - - # first, determine if we have an EasyConfig instance - # (indirectly by checking for 'iterating' and 'iterate_options' attributes, - # because we can't import the EasyConfig class here without introducing - # a cyclic import...); - # we need to know to determine whether we're iterating over a list of build dependencies - is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options') - if is_easyconfig: - # if we're iterating over different lists of build dependencies, - # only consider build dependencies when we're actually in iterative mode! - if 'builddependencies' in config.iterate_options: - if config.iterating: - build_deps = config.get('builddependencies') - else: - build_deps = None - else: + name_to_prefix = {name.lower(): prefix for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items()} + deps = config.get('dependencies', []) + + # also consider build dependencies for *ver and *shortver templates; + # we need to be a bit careful here, because for iterative installations + # (when multi_deps is used for example) the builddependencies value may be a list of lists + + # first, determine if we have an EasyConfig instance + # (indirectly by checking for 'iterating' and 'iterate_options' attributes, + # because we can't import the EasyConfig class here without introducing + # a cyclic import...); + # we need to know to determine whether we're iterating over a list of build dependencies + is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options') + if is_easyconfig: + # if we're iterating over different lists of build dependencies, + # only consider build dependencies when we're actually in iterative mode! + if 'builddependencies' in config.iterate_options: + if config.iterating: build_deps = config.get('builddependencies') + else: + build_deps = None + else: + build_deps = config.get('builddependencies') + if build_deps: + # Don't use += to avoid changing original list + deps = deps + build_deps + # include all toolchain deps (e.g. CUDAcore component in fosscuda); + # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain! + if config._toolchain is not None and config._toolchain.tcdeps: + # If we didn't create a new list above do it here if build_deps: - # Don't use += to avoid changing original list - deps = deps + build_deps - # include all toolchain deps (e.g. CUDAcore component in fosscuda); - # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain! - if config._toolchain is not None and config._toolchain.tcdeps: - # If we didn't create a new list above do it here - if build_deps: - deps.extend(config._toolchain.tcdeps) - else: - deps = deps + config._toolchain.tcdeps - - for dep in deps: - if isinstance(dep, dict): - dep_name, dep_version = dep['name'], dep['version'] - - # take into account dependencies marked as external modules, - # where name/version may have to be harvested from metadata available for that external module - if dep.get('external_module', False): - metadata = dep.get('external_module_metadata', {}) - if dep_name is None: - # name is a list in metadata, just take first value (if any) - dep_name = metadata.get('name', [None])[0] - if dep_version is None: - # version is a list in metadata, just take first value (if any) - dep_version = metadata.get('version', [None])[0] - - elif isinstance(dep, (list, tuple)): - dep_name, dep_version = dep[0], dep[1] + deps.extend(config._toolchain.tcdeps) else: - raise EasyBuildError("Unexpected type for dependency: %s", dep) - - if isinstance(dep_name, str) and dep_version: - pref = name_to_prefix.get(dep_name.lower()) - if pref: - dep_version = pick_dep_version(dep_version) - template_values['%sver' % pref] = dep_version - dep_version_parts = dep_version.split('.') - template_values['%smajver' % pref] = dep_version_parts[0] - if len(dep_version_parts) > 1: - template_values['%sminver' % pref] = dep_version_parts[1] - template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2]) + deps = deps + config._toolchain.tcdeps + + for dep in deps: + if isinstance(dep, dict): + dep_name, dep_version = dep['name'], dep['version'] + + # take into account dependencies marked as external modules, + # where name/version may have to be harvested from metadata available for that external module + if dep.get('external_module', False): + metadata = dep.get('external_module_metadata', {}) + if dep_name is None: + # name is a list in metadata, just take first value (if any) + dep_name = metadata.get('name', [None])[0] + if dep_version is None: + # version is a list in metadata, just take first value (if any) + dep_version = metadata.get('version', [None])[0] + + elif isinstance(dep, (list, tuple)): + dep_name, dep_version = dep[0], dep[1] + else: + raise EasyBuildError("Unexpected type for dependency: %s", dep) + + if isinstance(dep_name, str) and dep_version: + pref = name_to_prefix.get(dep_name.lower()) + if pref: + dep_version = pick_dep_version(dep_version) + template_values['%sver' % pref] = dep_version + dep_version_parts = dep_version.split('.') + template_values['%smajver' % pref] = dep_version_parts[0] + if len(dep_version_parts) > 1: + template_values['%sminver' % pref] = dep_version_parts[1] + template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2]) # step 3: add remaining from config for name in TEMPLATE_NAMES_CONFIG: @@ -464,8 +468,7 @@ def template_constant_dict(config, ignore=None, toolchain=None): unknown_names = [] for key in template_values: - dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC) - if not (key in common_template_names or key in dynamic_template_names): + if not (key in common_template_names or key in TEMPLATE_NAMES_DYNAMIC): unknown_names.append(key) if unknown_names: raise EasyBuildError("One or more template values found with unknown name: %s", ','.join(unknown_names)) @@ -515,15 +518,15 @@ def template_documentation(): # step 1: add TEMPLATE_NAMES_EASYCONFIG doc.append('Template names/values derived from easyconfig instance') - for name in TEMPLATE_NAMES_EASYCONFIG: - doc.append("%s%%(%s)s: %s" % (indent_l1, name[0], name[1])) + for name, cur_doc in TEMPLATE_NAMES_EASYCONFIG.items(): + doc.append("%s%%(%s)s: %s" % (indent_l1, name, cur_doc)) # step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS doc.append("Template names/values for (short) software versions") - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: - doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name)) - doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, pref, name)) - doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name)) + for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items(): + doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, prefix, name)) + doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, prefix, name)) + doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, prefix, name)) # step 3: add remaining self._config doc.append('Template names/values as set in easyconfig') @@ -539,11 +542,11 @@ def template_documentation(): # step 5. self.template_values can/should be updated from outside easyconfig # (eg the run_setp code in EasyBlock) doc.append('Template values set outside EasyBlock runstep') - for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - doc.append("%s%%(%s)s: %s" % (indent_l1, name[0], name[1])) + for name, cur_doc in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.items(): + doc.append("%s%%(%s)s: %s" % (indent_l1, name, cur_doc)) doc.append('Template constants that can be used in easyconfigs') - for cst in TEMPLATE_CONSTANTS: - doc.append('%s%s: %s (%s)' % (indent_l1, cst[0], cst[2], cst[1])) + for name, (value, cur_doc) in TEMPLATE_CONSTANTS.items(): + doc.append('%s%s: %s (%s)' % (indent_l1, name, cur_doc, value)) return "\n".join(doc) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index c6020706d5..90b9abaac2 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -116,7 +116,7 @@ def __init__(self, mself, ext, extra_params=None): # Add install/builddir templates with values from master. for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None)) + self.cfg.template_values[key] = str(getattr(self.master, key, None)) # We can't inherit the 'start_dir' value from the parent (which will be set, and will most likely be wrong). # It should be specified for the extension specifically, or be empty (so it is auto-derived). diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4bbdefa66b..b4d54685a9 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -478,16 +478,16 @@ def avail_easyconfig_templates_txt(): # step 1: add TEMPLATE_NAMES_EASYCONFIG doc.append('Template names/values derived from easyconfig instance') - for name in TEMPLATE_NAMES_EASYCONFIG: - doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + for name, curDoc in TEMPLATE_NAMES_EASYCONFIG.items(): + doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, curDoc)) doc.append('') # step 2: add SOFTWARE_VERSIONS doc.append('Template names/values for (short) software versions') - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: - doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name)) - doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, pref, name)) - doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name)) + for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items(): + doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, prefix, name)) + doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, prefix, name)) + doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, prefix, name)) doc.append('') # step 3: add remaining config @@ -506,20 +506,20 @@ def avail_easyconfig_templates_txt(): # step 5: template_values can/should be updated from outside easyconfig # (eg the run_step code in EasyBlock) doc.append('Template values set outside EasyBlock runstep') - for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + for name, cur_doc in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.items(): + doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc)) doc.append('') # some template values are only defined dynamically, # see template_constant_dict function in easybuild.framework.easyconfigs.templates doc.append('Template values which are defined dynamically') - for name in TEMPLATE_NAMES_DYNAMIC: - doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + for name, cur_doc in TEMPLATE_NAMES_DYNAMIC.items(): + doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc)) doc.append('') doc.append('Template constants that can be used in easyconfigs') - for cst in TEMPLATE_CONSTANTS: - doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, cst[0], cst[2], cst[1])) + for name, (value, doc) in TEMPLATE_CONSTANTS.items(): + doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, name, doc, value)) return '\n'.join(doc) @@ -530,8 +530,8 @@ def avail_easyconfig_templates_rst(): title = 'Template names/values derived from easyconfig instance' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG], - [name[1] for name in TEMPLATE_NAMES_EASYCONFIG], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG], + list(TEMPLATE_NAMES_EASYCONFIG.values()), ] doc = rst_title_and_table(title, table_titles, table_values) doc.append('') @@ -539,10 +539,10 @@ def avail_easyconfig_templates_rst(): title = 'Template names/values for (short) software versions' ver = [] ver_desc = [] - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: - ver.append('``%%(%smajver)s``' % pref) - ver.append('``%%(%sshortver)s``' % pref) - ver.append('``%%(%sver)s``' % pref) + for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items(): + ver.append('``%%(%smajver)s``' % prefix) + ver.append('``%%(%sshortver)s``' % prefix) + ver.append('``%%(%sver)s``' % prefix) ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (.)' % name) ver_desc.append('full version for %s' % name) @@ -565,24 +565,24 @@ def avail_easyconfig_templates_rst(): title = 'Template values set outside EasyBlock runstep' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], - [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()), ] doc.extend(rst_title_and_table(title, table_titles, table_values)) title = 'Template values which are defined dynamically' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC], - [name[1] for name in TEMPLATE_NAMES_DYNAMIC], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC], + list(TEMPLATE_NAMES_DYNAMIC.values()), ] doc.extend(rst_title_and_table(title, table_titles, table_values)) title = 'Template constants that can be used in easyconfigs' - titles = ['Constant', 'Template value', 'Template name'] + titles = ['Constant', 'Template description', 'Template value'] table_values = [ - ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS], - [cst[2] for cst in TEMPLATE_CONSTANTS], - ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS], + ['``%s``' % name for name in TEMPLATE_CONSTANTS], + [doc for _, doc in TEMPLATE_CONSTANTS.values()], + ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()], ] doc.extend(rst_title_and_table(title, titles, table_values)) @@ -595,8 +595,8 @@ def avail_easyconfig_templates_md(): title = 'Template names/values derived from easyconfig instance' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG], - [name[1] for name in TEMPLATE_NAMES_EASYCONFIG], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG], + list(TEMPLATE_NAMES_EASYCONFIG.values()), ] doc = md_title_and_table(title, table_titles, table_values, title_level=2) doc.append('') @@ -604,10 +604,10 @@ def avail_easyconfig_templates_md(): title = 'Template names/values for (short) software versions' ver = [] ver_desc = [] - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: - ver.append('``%%(%smajver)s``' % pref) - ver.append('``%%(%sshortver)s``' % pref) - ver.append('``%%(%sver)s``' % pref) + for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items(): + ver.append('``%%(%smajver)s``' % prefix) + ver.append('``%%(%sshortver)s``' % prefix) + ver.append('``%%(%sver)s``' % prefix) ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (``.``)' % name) ver_desc.append('full version for %s' % name) @@ -631,27 +631,28 @@ def avail_easyconfig_templates_md(): title = 'Template values set outside EasyBlock runstep' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], - [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()), ] doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) doc.append('') title = 'Template values which are defined dynamically' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC], - [name[1] for name in TEMPLATE_NAMES_DYNAMIC], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC], + list(TEMPLATE_NAMES_DYNAMIC.values()), ] doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) doc.append('') title = 'Template constants that can be used in easyconfigs' - titles = ['Constant', 'Template value', 'Template name'] + titles = ['Constant', 'Template description', 'Template value'] table_values = [ - ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS], - [cst[2] for cst in TEMPLATE_CONSTANTS], - ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS], + ['``%s``' % name for name in TEMPLATE_CONSTANTS], + [doc for _, doc in TEMPLATE_CONSTANTS.values()], + ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()], ] + doc.extend(md_title_and_table(title, titles, table_values, title_level=2)) return '\n'.join(doc) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 67edcd271e..ff54451955 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1378,7 +1378,7 @@ def test_templating_doc(self): # expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between) temps = [ easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG, - easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3, + list(easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS.keys()) * 3, easyconfig.templates.TEMPLATE_NAMES_CONFIG, easyconfig.templates.TEMPLATE_NAMES_LOWER, easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, From 9281587caee990d6befaf2579a31fe6a3c607776 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 5 Aug 2024 12:51:35 +0200 Subject: [PATCH 387/430] allow import of template contants by name This allows direct access to the template constants, e.g. for use in hooks. E.g.: `from easybuild.framework.easyconfig.templates import GITHUB_SOURCE` --- easybuild/framework/easyconfig/templates.py | 5 +++++ easybuild/tools/docs.py | 4 ++-- test/framework/easyconfig.py | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index f6df89e682..9d1b257fdf 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -550,3 +550,8 @@ def template_documentation(): doc.append('%s%s: %s (%s)' % (indent_l1, name, cur_doc, value)) return "\n".join(doc) + + +# Add template constants to export list +globals().update({name: value for name, (value, _) in TEMPLATE_CONSTANTS.items()}) +__all__ = list(TEMPLATE_CONSTANTS.keys()) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index b4d54685a9..69e8d17337 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -518,8 +518,8 @@ def avail_easyconfig_templates_txt(): doc.append('') doc.append('Template constants that can be used in easyconfigs') - for name, (value, doc) in TEMPLATE_CONSTANTS.items(): - doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, name, doc, value)) + for name, (value, cur_doc) in TEMPLATE_CONSTANTS.items(): + doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, name, cur_doc, value)) return '\n'.join(doc) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index ff54451955..779d8b0cc2 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1271,6 +1271,14 @@ def test_templating_constants(self): ec = EasyConfig(test_ec) self.assertEqual(ec['sanity_check_commands'], ['mpiexec -np 1 -- toy']) + def test_template_constant_import(self): + """Test importing template constants works""" + from easybuild.framework.easyconfig.templates import GITHUB_SOURCE, GNU_SOURCE, SHLIB_EXT + from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS + self.assertEqual(GITHUB_SOURCE, TEMPLATE_CONSTANTS['GITHUB_SOURCE'][0]) + self.assertEqual(GNU_SOURCE, TEMPLATE_CONSTANTS['GNU_SOURCE'][0]) + self.assertEqual(SHLIB_EXT, get_shared_lib_ext()) + def test_templating_cuda_toolchain(self): """Test templates via toolchain component, like setting %(cudaver)s with fosscuda toolchain.""" From d00590ccf8bdfa82f3ec4f6c1da971d3d95fa628 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 5 Aug 2024 10:11:34 +0200 Subject: [PATCH 388/430] add GNU_FTP_SOURCE template constant `GNU_SOURCE` (https://ftpmirror.gnu.org/gnu) is sometimes unreachable. We can use the main source instead of the mirror: https://ftp.gnu.org/gnu Add the constant `GNU_FTP_SOURCE` for that. Recommended use: `source_urls = [GNU_SOURCE, GNU_FTP_SOURCE]` --- easybuild/framework/easyconfig/templates.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 0e2958ede7..00aeaf4d38 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -132,7 +132,9 @@ ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s', 'download.savannah.gnu.org source url'), ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s', - 'gnu.org source url'), + 'gnu.org source url (ftp mirror)'), + ('GNU_FTP_SOURCE', 'https://ftp.gnu.org/gnu/%(namelower)s', + 'gnu.org source url (main ftp)'), ('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files', 'googlecode.com source url'), ('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/', From 521ab009e1cfcfd50418e7bc51992145c5d424e0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 5 Aug 2024 13:48:36 +0200 Subject: [PATCH 389/430] Add GNU_FTP_URL alternative --- easybuild/framework/easyconfig/templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 00aeaf4d38..40885343f9 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -218,6 +218,7 @@ 'GITHUB_RELEASE_URL': 'GITHUB_RELEASE', 'GITHUB_RELEASE_URL_LOWER': 'GITHUB_LOWER_RELEASE', 'GNU_SAVANNAH_URL': 'GNU_SAVANNAH_SOURCE', + 'GNU_FTP_URL': 'GNU_FTP_SOURCE', 'GNU_URL': 'GNU_SOURCE', 'GOOGLECODE_URL': 'GOOGLECODE_SOURCE', 'LAUNCHPAD_URL': 'LAUNCHPAD_SOURCE', From 584755115611a2fbe34d324d9947736224c9b50f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 5 Aug 2024 15:02:06 +0200 Subject: [PATCH 390/430] fix test_toy_lock_cleanup_signals The 1s timeout might not be enough for the build to create the log (e.g. when ran in a debugger) Increase the timeout and also pass required arguments to actually create the lock. --- test/framework/toy_build.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 2978e64b1a..97e154b23f 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3753,7 +3753,9 @@ def __exit__(self, type, value, traceback): toy_ec_txt = read_file(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, toy_ec_txt + '\npostinstallcmds = ["sleep 5"]') + write_file(test_ec, toy_ec_txt + '\npostinstallcmds = ["sleep 10"]') + + extra_args = ['--locks-dir=%s' % locks_dir, '--wait-on-lock-limit=3', '--wait-on-lock-interval=3'] signums = [ (signal.SIGABRT, SystemExit), @@ -3766,7 +3768,7 @@ def __exit__(self, type, value, traceback): # avoid recycling stderr of previous test stderr = '' - with WaitAndSignal(1, signum): + with WaitAndSignal(3, signum): # change back to original working directory before each test change_dir(orig_wd) @@ -3774,7 +3776,7 @@ def __exit__(self, type, value, traceback): self.mock_stderr(True) self.mock_stdout(True) self.assertErrorRegex(exc, '.*', self._test_toy_build, ec_file=test_ec, verify=False, - raise_error=True, testing=False, raise_systemexit=True) + extra_args=extra_args, raise_error=True, testing=False, raise_systemexit=True) stderr = self.get_stderr().strip() self.mock_stderr(False) From 475d0afef819f97544454f25a6747243585df867 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 8 Aug 2024 08:42:38 +0100 Subject: [PATCH 391/430] limit `maxparallel` to 16 by default --- easybuild/framework/easyconfig/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index aedb9edfa1..438bcea4e1 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -108,7 +108,7 @@ BUILD], 'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD], 'installopts': ['', 'Extra options for installation', BUILD], - 'maxparallel': [None, 'Max degree of parallelism', BUILD], + 'maxparallel': [16, 'Max degree of parallelism', BUILD], 'module_only': [False, 'Only generate module file', BUILD], 'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of ' 'cores, active cpuset and restrictions in ulimit)'), BUILD], From ee9ca65c57a384fe5e1cfb453c3a30546dfe2c95 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Jul 2021 14:34:32 +0200 Subject: [PATCH 392/430] Really remove support code for Python < 2.7 --- .github/workflows/unit_tests.yml | 6 +++--- easybuild/tools/systemtools.py | 1 - test/framework/repository.py | 6 +----- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 1072bf421d..6fb76814cf 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -101,12 +101,12 @@ jobs: # and are only run after the PR gets merged GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} run: | - # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit; + # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit # tests that require a GitHub token are skipped automatically when no GitHub token is available if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then if [ ! -z $GITHUB_TOKEN ]; then - SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; - python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())" + python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')" fi echo "GitHub token installed!" else diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index b4e1c254da..2fdd9c4ba8 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -736,7 +736,6 @@ def get_os_name(): # platform.linux_distribution was removed in Python 3.8, # see https://docs.python.org/2/library/platform.html#platform.linux_distribution if hasattr(platform, 'linux_distribution'): - # platform.linux_distribution is more useful, but only available since Python 2.6 # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=PendingDeprecationWarning) diff --git a/test/framework/repository.py b/test/framework/repository.py index 808f9fd43c..728039bf62 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -129,11 +129,7 @@ def test_svnrepo(self): self.assertExists(os.path.join(repo.wc, 'trunk', 'README.md')) shutil.rmtree(repo.wc) - # this test is disabled because it fails in Travis as a result of bitbucket disabling TLS 1.0/1.1 - # we can consider re-enabling it when moving to a more recent Ubuntu version in the Travis config - # (which implies dropping support for Python 2.6) - # cfr. https://github.com/easybuilders/easybuild-framework/pull/2678 - def DISABLED_test_hgrepo(self): + def test_hgrepo(self): """Test using HgRepository.""" # only run this test if pysvn Python module is available try: From e2dea1fb16d422ab22986f34feb1dc46364fec8a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Jul 2021 14:37:14 +0200 Subject: [PATCH 393/430] Use OrderedDict from collections not py2vs3 This is from the python installation since 2.7 --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/tools/py2vs3/py3.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 772057fa6b..ae678f0269 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -46,8 +46,8 @@ import functools import os import re -from contextlib import contextmanager from collections import OrderedDict +from contextlib import contextmanager import easybuild.tools.filetools as filetools from easybuild.base import fancylogger diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index a01cb64d39..2f29f0fd54 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -36,7 +36,6 @@ import json import sys import urllib.request as std_urllib # noqa -from collections import OrderedDict # noqa from collections.abc import Mapping # noqa from functools import cmp_to_key from importlib.util import spec_from_file_location, module_from_spec From 71b7b214acc9d0be643509e8f8979bd2421972c2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Jul 2021 15:38:08 +0200 Subject: [PATCH 394/430] Remove outdated Mercurial test --- test/framework/repository.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/framework/repository.py b/test/framework/repository.py index 728039bf62..b45d5d7165 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -40,7 +40,6 @@ from easybuild.tools.filetools import read_file from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.repository.gitrepo import GitRepository -from easybuild.tools.repository.hgrepo import HgRepository from easybuild.tools.repository.svnrepo import SvnRepository from easybuild.tools.repository.repository import init_repository from easybuild.tools.run import run_shell_cmd @@ -129,23 +128,6 @@ def test_svnrepo(self): self.assertExists(os.path.join(repo.wc, 'trunk', 'README.md')) shutil.rmtree(repo.wc) - def test_hgrepo(self): - """Test using HgRepository.""" - # only run this test if pysvn Python module is available - try: - import hglib # noqa - except ImportError: - print("(skipping HgRepository test)") - return - - # GitHub also supports SVN - test_repo_url = 'https://kehoste@bitbucket.org/kehoste/testrepository' - - repo = HgRepository(test_repo_url) - repo.init() - self.assertExists(os.path.join(repo.wc, 'README')) - shutil.rmtree(repo.wc) - def test_init_repository(self): """Test use of init_repository function.""" repo = init_repository('FileRepository', self.path) From 6b4e7ba4a3704a390aee1c95057ef07d6a762103 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 3 Aug 2021 16:34:51 +0200 Subject: [PATCH 395/430] Simplify ignore patterns --- .github/workflows/unit_tests.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 6fb76814cf..5bf29eb164 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -187,7 +187,17 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test|CryptographyDeprecationWarning: TripleDES has been moved|algorithms.TripleDES" + IGNORE_PATTERNS="no GitHub token available" + IGNORE_PATTERNS+="|skipping SvnRepository test" + IGNORE_PATTERNS+="|requires Lmod as modules tool" + IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device" + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]" + IGNORE_PATTERNS+="|from cryptography.* import " + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2" + IGNORE_PATTERNS+="|Blowfish" + IGNORE_PATTERNS+="|GC3Pie not available, skipping test" + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved" + IGNORE_PATTERNS+="|algorithms.TripleDES" # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) From dcbcbeb1e2f9ef7404283c5a1b45e3fe3e1a4168 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 8 Aug 2024 10:51:20 +0100 Subject: [PATCH 396/430] alter tests --- test/framework/easyblock.py | 4 ++-- test/framework/easyconfig.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 30b95c18db..730b0ba180 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2184,7 +2184,7 @@ def test_parallel(self): handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec1, toytxt + "\nparallel = 123") + write_file(toy_ec1, toytxt + "\nparallel = 13") handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) @@ -2202,7 +2202,7 @@ def test_parallel(self): # only 'parallel' easyconfig parameter specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec1)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 123) + self.assertEqual(test_eb.cfg['parallel'], 13) # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec2)) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 67edcd271e..b3acedd1f2 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3476,7 +3476,7 @@ def test_template_constant_dict(self): except AttributeError: pass # Ignore if not present orig_get_avail_core_count = st.get_avail_core_count - st.get_avail_core_count = lambda: 42 + st.get_avail_core_count = lambda: 12 # also check template values after running check_readiness_step (which runs set_parallel) eb = EasyBlock(ec) @@ -3487,7 +3487,7 @@ def test_template_constant_dict(self): res = template_constant_dict(ec) res.pop('arch') - expected['parallel'] = 42 + expected['parallel'] = 12 self.assertEqual(res, expected) toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb') From 013e891db8de46b49fe64e171c4c86c055c3ca56 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:42:43 +0100 Subject: [PATCH 397/430] fix tests --- test/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 730b0ba180..7668413f8d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2188,7 +2188,7 @@ def test_parallel(self): handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67") + write_file(toy_ec2, toytxt + "\nparallel = 14\nmaxparallel = 7") handle, toy_ec3 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) @@ -2207,7 +2207,7 @@ def test_parallel(self): # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 67) + self.assertEqual(test_eb.cfg['parallel'], 7) # make sure 'parallel = False' is not overriden (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec3)) @@ -2215,20 +2215,20 @@ def test_parallel(self): self.assertEqual(test_eb.cfg['parallel'], False) # only 'parallel' build option specified - init_config(build_options={'parallel': '97', 'validate': False}) + init_config(build_options={'parallel': '9', 'validate': False}) test_eb = EasyBlock(EasyConfig(toy_ec)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 97) + self.assertEqual(test_eb.cfg['parallel'], 9) # both 'parallel' build option and easyconfig parameter specified (no 'maxparallel') test_eb = EasyBlock(EasyConfig(toy_ec1)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 97) + self.assertEqual(test_eb.cfg['parallel'], 9) # both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 67) + self.assertEqual(test_eb.cfg['parallel'], 7) # make sure 'parallel = False' is not overriden (with 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec3)) From db97a36b19797fe157902251512f2c3f1fd41dad Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:21:20 +0100 Subject: [PATCH 398/430] revert unneeded changes --- test/framework/easyblock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 7668413f8d..01014e3ee1 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2188,7 +2188,7 @@ def test_parallel(self): handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec2, toytxt + "\nparallel = 14\nmaxparallel = 7") + write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67") handle, toy_ec3 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) @@ -2207,7 +2207,7 @@ def test_parallel(self): # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 7) + self.assertEqual(test_eb.cfg['parallel'], 67) # make sure 'parallel = False' is not overriden (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec3)) @@ -2228,7 +2228,7 @@ def test_parallel(self): # both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 7) + self.assertEqual(test_eb.cfg['parallel'], 9) # make sure 'parallel = False' is not overriden (with 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec3)) From c0e1aae337ab0506feb22cacf13711c1519ddf01 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 9 Aug 2024 17:53:19 +0200 Subject: [PATCH 399/430] Add support for `--software-commit=23be34` and an associated template --- easybuild/framework/easyconfig/templates.py | 6 ++++- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 9 +++++++ test/framework/easyconfig.py | 29 +++++++++++++++++++++ test/framework/options.py | 16 ++++++++++++ 5 files changed, 60 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 0e2958ede7..929f65a1a4 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -90,7 +90,7 @@ TEMPLATE_NAMES_DYNAMIC = [ ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" - "as specify by the --sysroot configuration option"), + "as specified by the --sysroot configuration option"), ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " "--cuda-compute-capabilities configuration option or via cuda_cc easyconfig parameter"), @@ -104,6 +104,7 @@ ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"), ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), + ('software_commit', "Git commit id to use for the software as specified by --software-commit command line option"), ] # constant templates that can be used in easyconfigs @@ -294,6 +295,9 @@ def template_constant_dict(config, ignore=None, toolchain=None): # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback template_values['sysroot'] = build_option('sysroot') or '' + # set 'software_commit' template based on 'software_commit' configuration option, default to None + template_values['software_commit'] = build_option('software_commit') or '' + # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: if name in ignore: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 1864ec0dc0..54c2c593af 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -260,6 +260,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'rpath_override_dirs', 'required_linked_shared_libs', 'skip', + 'software_commit', 'stop', 'subdir_user_modules', 'sysroot', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index bdeb4890bd..956f9b883d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -509,6 +509,9 @@ def override_options(self): 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), + 'software-commit': ( + "Git commit to use for the target software build (robot capabilities are automatically disabled)", + None, 'store', None), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include", None, 'store', None), @@ -1092,6 +1095,12 @@ def _postprocess_checks(self): else: raise EasyBuildError("Specified sysroot '%s' does not exist!", self.options.sysroot) + # 'software_commit' is specific to a particular software package and cannot be used with 'robot' + if self.options.software_commit: + if self.options.robot is not None: + print_warning("To allow use of --software-commit robot resolution is being disabled") + self.options.robot = None + self.log.info("Checks on configuration options passed") def get_cfg_opt_abs_path(self, opt_name, path): diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index b3acedd1f2..8768c2adc2 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1462,6 +1462,35 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) + def test_software_commit_template(self): + """Test the %(software_commit)s template""" + + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nconfigopts = "--some-opt=%(software_commit)s"' + test_ec_txt += '\nbuildopts = "--some-opt=%(software_commit)s"' + test_ec_txt += '\ninstallopts = "--some-opt=%(software_commit)s"' + write_file(test_ec, test_ec_txt) + + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=") + self.assertEqual(ec['buildopts'], "--some-opt=") + self.assertEqual(ec['installopts'], "--some-opt=") + + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist + software_commit = '1234bc' + update_build_option('software_commit', software_commit) + + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=%s" % software_commit) + self.assertEqual(ec['buildopts'], "--some-opt=%s" % software_commit) + self.assertEqual(ec['installopts'], "--some-opt=%s" % software_commit) + def test_template_deprecation_and_alternative(self): """Test deprecation of (and alternative) templates""" diff --git a/test/framework/options.py b/test/framework/options.py index 8a2146d2be..fe2a6ac0dc 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6906,6 +6906,22 @@ def test_sysroot(self): os.environ['EASYBUILD_SYSROOT'] = doesnotexist self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) + def test_software_commit(self): + """Test use of --software-commit option.""" + + software_commit = "23be34" + software_commit_arg = '--software-commit=' + software_commit + # Add robot to also test that it gets disabled + stdout, stderr = self._run_mock_eb([software_commit_arg, '--show-config', '--robot'], raise_error=True) + + warning_regex = re.compile(r'.*WARNING:.*--software-commit robot resolution is being disabled.*', re.M) + software_commit_regex = re.compile(r'^software-commit\s*\(C\) = %s$' % software_commit, re.M) + robot_regex = re.compile(r'^robot\s*\(C\) = .*', re.M) + + self.assertTrue(warning_regex.search(stderr), "Pattern '%s' not found in: %s" % (warning_regex, stderr)) + self.assertTrue(software_commit_regex.search(stdout), "Pattern '%s' not found in: %s" % (software_commit_regex, stdout)) + self.assertFalse(robot_regex.search(stdout), "Pattern '%s' found in: %s" % (robot_regex, stdout)) + def test_accept_eula_for(self): """Test --accept-eula-for configuration option.""" From 14748e330f6cc93dd61d78a0f17254a776e75ace Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 9 Aug 2024 17:56:00 +0200 Subject: [PATCH 400/430] Fix lint --- test/framework/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index fe2a6ac0dc..5739527341 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6919,7 +6919,8 @@ def test_software_commit(self): robot_regex = re.compile(r'^robot\s*\(C\) = .*', re.M) self.assertTrue(warning_regex.search(stderr), "Pattern '%s' not found in: %s" % (warning_regex, stderr)) - self.assertTrue(software_commit_regex.search(stdout), "Pattern '%s' not found in: %s" % (software_commit_regex, stdout)) + self.assertTrue(software_commit_regex.search(stdout), + "Pattern '%s' not found in: %s" % (software_commit_regex, stdout)) self.assertFalse(robot_regex.search(stdout), "Pattern '%s' found in: %s" % (robot_regex, stdout)) def test_accept_eula_for(self): From cf9005fda2e53a4d3b9df723bef1f138245d1e0e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 9 Aug 2024 18:49:33 +0200 Subject: [PATCH 401/430] Fix other tests --- test/framework/easyconfig.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 8768c2adc2..87ee6381a2 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3480,6 +3480,7 @@ def test_template_constant_dict(self): 'nameletter': 'g', 'nameletterlower': 'g', 'parallel': None, + 'software_commit': '', 'sysroot': '', 'toolchain_name': 'foss', 'toolchain_version': '2018a', @@ -3563,6 +3564,7 @@ def test_template_constant_dict(self): 'pyminver': '7', 'pyshortver': '3.7', 'pyver': '3.7.2', + 'software_commit': '', 'sysroot': '', 'version': '0.01', 'version_major': '0', @@ -3628,6 +3630,7 @@ def test_template_constant_dict(self): 'namelower': 'foo', 'nameletter': 'f', 'nameletterlower': 'f', + 'software_commit': '', 'sysroot': '', 'version': '1.2.3', 'version_major': '1', From 2cc94e9f1afd3fd02ad2b34e11af48117c6c658f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Aug 2024 17:20:15 +0200 Subject: [PATCH 402/430] improve logging in helper function used by run_shell_cmd to answer questions for interactive shell commands --- easybuild/tools/run.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index fbfb36dc74..30ddc307c6 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -257,6 +257,7 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): # replace spaces/line breaks with regex pattern that matches one or more spaces/line breaks, # and allow extra whitespace at the end question = space_line_break_pattern.join(space_line_break_regex.split(question)) + r'[\s\n]*$' + _log.debug(f"Checking for question pattern '{question}'...") regex = re.compile(question.encode()) res = regex.search(stdout) if res: @@ -284,6 +285,8 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): match_found = True break + else: + _log.debug(f"No match for question pattern '{question}' at end of stdout: {stdout_end}") else: _log.info("No match found for question patterns, considering question wait patterns") # if no match was found among question patterns, @@ -295,14 +298,17 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): # and allow extra whitespace at the end pattern = space_line_break_pattern.join(space_line_break_regex.split(pattern)) + r'[\s\n]*$' regex = re.compile(pattern.encode()) + _log.debug(f"Checking for question wait pattern '{pattern}'...") if regex.search(stdout): - _log.info(f"Found match for wait pattern '{pattern}'") - _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout_end}") + _log.info(f"Found match for question wait pattern '{pattern}'") + _log.debug(f"Found match for question wait pattern '{pattern}' at end of stdout: {stdout_end}") match_found = True break + else: + _log.debug(f"No match for question wait pattern '{pattern}' at end of stdout: {stdout_end}") else: _log.info("No match found for question wait patterns") - _log.debug(f"No match found in question/wait patterns at end of stdout: {stdout_end}") + _log.debug(f"No match found in question (wait) patterns at end of stdout: {stdout_end}") return match_found From 6ab51969fc5932aeaf4a94c5d202f4c457ff3209 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Aug 2024 19:12:13 +0200 Subject: [PATCH 403/430] let cmd.sh script produced by run_shell_cmd define $EB_CMD_OUT_FILE + $EB_CMD_ERR_FILE --- easybuild/tools/run.py | 12 +++++++++--- test/framework/run.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index fbfb36dc74..d5e6b859a3 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -196,7 +196,7 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): return ''.join([c for c in cmd if c in allowed_chars]) -def create_cmd_scripts(cmd_str, work_dir, env, tmpdir): +def create_cmd_scripts(cmd_str, work_dir, env, tmpdir, out_file, err_file): """ Create helper scripts for specified command in specified directory: - env.sh which can be sourced to define environment in which command was run; @@ -220,6 +220,12 @@ def create_cmd_scripts(cmd_str, work_dir, env, tmpdir): fid.write('\n\nPS1="eb-shell> "') + # define $EB_CMD_OUT_FILE to contain path to file with command output + fid.write(f'\nEB_CMD_OUT_FILE="{out_file}"') + # define $EB_CMD_ERR_FILE to contain path to file with command stderr output (if available) + if err_file: + fid.write(f'\nEB_CMD_ERR_FILE="{err_file}"') + # also change to working directory (to ensure that working directory is correct for interactive bash shell) fid.write(f'\ncd "{work_dir}"') @@ -402,8 +408,6 @@ def to_cmd_str(cmd): _log.info(f'run_shell_cmd: command environment of "{cmd_str}" will be saved to {tmpdir}') - create_cmd_scripts(cmd_str, work_dir, env, tmpdir) - cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_shell_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') if split_stderr: @@ -411,6 +415,8 @@ def to_cmd_str(cmd): _log.info(f'run_shell_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') else: cmd_err_fp = None + + create_cmd_scripts(cmd_str, work_dir, env, tmpdir, cmd_out_fp, cmd_err_fp) else: tmpdir, cmd_out_fp, cmd_err_fp = None, None, None diff --git a/test/framework/run.py b/test/framework/run.py index ab0d67e289..1c92d19903 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -51,7 +51,7 @@ import easybuild.tools.utilities from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging from easybuild.tools.config import update_build_option -from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file +from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, remove_dir, write_file from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors from easybuild.tools.run import complete_cmd, fileprefix_from_cmd, get_output_from_process, parse_log_for_error from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate @@ -219,10 +219,13 @@ def test_run_shell_cmd_basic(self): cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh') self.assertExists(cmd_script) + cmd = f"{cmd_script} -c 'echo pwd: $PWD; echo $FOOBAR; echo $EB_CMD_OUT_FILE; cat $EB_CMD_OUT_FILE'" with self.mocked_stdout_stderr(): - res = run_shell_cmd(f"{cmd_script} -c 'echo pwd: $PWD; echo $FOOBAR'", fail_on_error=False) + res = run_shell_cmd(cmd, fail_on_error=False) self.assertEqual(res.exit_code, 0) - self.assertTrue(res.output.endswith('foobar\n')) + regex = re.compile("pwd: .*\nfoobar\n.*/echo-.*/out.txt\nhello$") + self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' should be found in {res.output}") + # check whether working directory is what's expected regex = re.compile('^pwd: .*', re.M) res = regex.findall(res.output) @@ -667,12 +670,36 @@ def test_run_shell_cmd_split_stderr(self): self.assertTrue("warning" in output_lines) self.assertEqual(res.stderr, None) + # cleanup of artifacts in between calls to run_shell_cmd + remove_dir(self.test_prefix) + with self.mocked_stdout_stderr(): res = run_shell_cmd(cmd, split_stderr=True) self.assertEqual(res.exit_code, 0) self.assertEqual(res.stderr, "warning\n") self.assertEqual(res.output, "ok\n") + # check whether environment variables that point to stdout/stderr output files + # are set in environment defined by cmd.sh script + paths = glob.glob(os.path.join(self.test_prefix, 'eb-*', 'run-shell-cmd-output', 'echo-*')) + self.assertEqual(len(paths), 1) + cmd_tmpdir = paths[0] + cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh') + self.assertExists(cmd_script) + + cmd_cmd = '; '.join([ + "echo $EB_CMD_OUT_FILE", + "cat $EB_CMD_OUT_FILE", + "echo $EB_CMD_ERR_FILE", + "cat $EB_CMD_ERR_FILE", + ]) + cmd = f"{cmd_script} -c '{cmd_cmd}'" + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, fail_on_error=False) + + regex = re.compile(".*/echo-.*/out.txt\nok\n.*/echo-.*/err.txt\nwarning$") + self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' should be found in {res.output}") + def test_run_cmd_trace(self): """Test run_cmd in trace mode, and with tracing disabled.""" From 64580f6da6c2744241f2038fa8fbdc6869794dd9 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sat, 4 Nov 2023 08:09:30 +0100 Subject: [PATCH 404/430] Use getenv modulefile command with EnvironmentModules Update "getenv_cmd" function from ModuleGeneratorTcl class to rely on "getenv" modulefile command on EnvironmentModules. "getenv" returns environment variable value or a fallback value if it does not exist. No error is thus raised if variable does not exist compared to using "$::env(VARNAME)" syntax. "getenv" modulefile command also has the advantage to return "$VARNAME" rather VARNAME value if modulefile is evaluated in "display" mode. Which helps to understand that an environment variable is relying on another one. Note that Lmod introduced a "getenv" Tcl modulefile command in version 8.7.31 but it does not handle fallback value at the moment. So the use of "getenv" with Lmod is not enabled at the moment, but proposed code in this change makes it easy to enable such support in the future. --- easybuild/tools/module_generator.py | 18 ++++++++++++------ easybuild/tools/modules.py | 6 ++++++ test/framework/module_generator.py | 23 +++++++++++++++++------ 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index c537003de5..9b1ee024c4 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -863,13 +863,19 @@ def getenv_cmd(self, envvar, default=None): Return module-syntax specific code to get value of specific environment variable. """ if default is None: - cmd = '$::env(%s)' % envvar + if self.modules_tool.supports_tcl_getenv: + cmd = '[getenv %s]' % envvar + else: + cmd = '$::env(%s)' % envvar else: - values = { - 'default': default, - 'envvar': '::env(%s)' % envvar, - } - cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values + if self.modules_tool.supports_tcl_getenv: + cmd = '[getenv %s "%s"]' % (envvar, default) + else: + values = { + 'default': default, + 'envvar': '::env(%s)' % envvar, + } + cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values return cmd def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_modules=None, multi_dep_mods=None): diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 846916ee62..14891847cb 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -148,6 +148,8 @@ class ModulesTool(object): VERSION_OPTION = '--version' # minimal required version (cannot include -beta or rc) REQ_VERSION = None + # minimal required version to use getenv Tcl modulefile command + REQ_VERSION_TCL_GETENV = None # deprecated version limit (support for versions below this version is deprecated) DEPR_VERSION = None # maximum version allowed (cannot include -beta or rc) @@ -209,6 +211,7 @@ def __init__(self, mod_paths=None, testing=False): self.check_module_function(allow_mismatch=build_option('allow_modules_tool_mismatch')) self.set_and_check_version() self.supports_depends_on = False + self.supports_tcl_getenv = False def __str__(self): """String representation of this ModulesTool instance.""" @@ -1315,6 +1318,7 @@ class EnvironmentModules(EnvironmentModulesTcl): COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' + REQ_VERSION_TCL_GETENV = '4.2.0' DEPR_VERSION = '4.0.0' # needs to be set as EnvironmentModules inherits from EnvironmentModulesTcl MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' @@ -1343,6 +1347,8 @@ def __init__(self, *args, **kwargs): setvar('MODULES_LIST_TERSE_OUTPUT', '', verbose=False) super(EnvironmentModules, self).__init__(*args, **kwargs) + version = LooseVersion(self.version) + self.supports_tcl_getenv = version >= LooseVersion(self.REQ_VERSION_TCL_GETENV) def check_module_function(self, allow_mismatch=False, regex=None): """Check whether selected module tool matches 'module' function definition.""" diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 075cc3adbd..015ab6520a 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -844,7 +844,10 @@ def test_det_user_modpath(self): init_config(build_options={'suffix_modules_path': ''}) user_modpath = 'my/{RUNTIME_ENV::TEST123}/modules' if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: - self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" $::env(TEST123) "modules"') + if self.modtool.supports_tcl_getenv: + self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" [getenv TEST123] "modules"') + else: + self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" $::env(TEST123) "modules"') else: self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", os.getenv("TEST123"), "modules"') @@ -902,12 +905,20 @@ def test_getenv_cmd(self): # otherwise we won't get the output produced by the test module file... os.environ.pop('LMOD_QUIET', None) - self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME')) - self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME')) + if self.modtool.supports_tcl_getenv: + self.assertEqual('[getenv HOSTNAME]', self.modgen.getenv_cmd('HOSTNAME')) + self.assertEqual('[getenv HOME]', self.modgen.getenv_cmd('HOME')) - expected = '[if { [info exists ::env(TEST)] } { concat $::env(TEST) } else { concat "foobar" } ]' - getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') - self.assertEqual(getenv_txt, expected) + expected = '[getenv TEST "foobar"]' + getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') + self.assertEqual(getenv_txt, expected) + else: + self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME')) + self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME')) + + expected = '[if { [info exists ::env(TEST)] } { concat $::env(TEST) } else { concat "foobar" } ]' + getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') + self.assertEqual(getenv_txt, expected) write_file(test_mod_file, '#%%Module\nputs stderr %s' % getenv_txt) else: From 0093c9e906d47df0fd569722b59a0fe93d1e1fff Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Aug 2024 19:45:07 +0200 Subject: [PATCH 405/430] fix run_shell_cmd to avoid that same question is answered multiple times --- easybuild/tools/run.py | 10 +++++++++- test/framework/run.py | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 1e5d4864b8..40f25bac68 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -489,6 +489,7 @@ def to_cmd_str(cmd): stdout, stderr = b'', b'' check_interval_secs = 0.1 time_no_match = 0 + prev_stdout = '' # collect output piece-wise, while checking for questions to answer (if qa_patterns is provided) while exit_code is None: @@ -514,8 +515,15 @@ def to_cmd_str(cmd): stderr += more_stderr if qa_patterns: - if _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): + # only check for question patterns if additional output is available + # compared to last time a question was answered; + # use empty list of question patterns iif no extra output (except for whitespace) is available + # we do always need to check for wait patterns though! + active_qa_patterns = qa_patterns if stdout.strip() != prev_stdout else [] + + if _answer_question(stdout, proc, active_qa_patterns, qa_wait_patterns): time_no_match = 0 + prev_stdout = stdout.strip() else: # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs diff --git a/test/framework/run.py b/test/framework/run.py index 1c92d19903..dff682c836 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1069,6 +1069,28 @@ def test_run_shell_cmd_qa(self): self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, "please\nanswer\n42\n") + # test interactive command that takes a while before producing more output that includes second question + cmd = ';'.join([ + "echo question1", + "read answer1", + "sleep 2", + "echo question2", + "read answer2", + # note: delaying additional output (except the actual questions) is important + # to verify that this is working as intended + "echo $answer1", + "echo $answer2", + ]) + qa = [ + (r'question1', 'answer1'), + (r'question2', 'answer2'), + ] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "question1\nquestion2\nanswer1\nanswer2\n") + def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" From c2e096b218e08b90f009386a3ec4fa5cebecf976 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 21 Aug 2024 08:37:42 +0100 Subject: [PATCH 406/430] fix typo --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 40f25bac68..b9bb2ed8c8 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -517,7 +517,7 @@ def to_cmd_str(cmd): if qa_patterns: # only check for question patterns if additional output is available # compared to last time a question was answered; - # use empty list of question patterns iif no extra output (except for whitespace) is available + # use empty list of question patterns if no extra output (except for whitespace) is available # we do always need to check for wait patterns though! active_qa_patterns = qa_patterns if stdout.strip() != prev_stdout else [] From 78d8d9dbc83ba9fb690bdee6782aca8180fe6f01 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sat, 4 Nov 2023 14:37:46 +0100 Subject: [PATCH 407/430] Add module cache build support on EnvironmentModules Environment Modules 5.3.0 introduced module cache mechanism. A cache file is generated with cachebuild command and stored under modulepath directory in a .modulecache file. Add cache build support in "update" function of EnvironmentModules class. --- easybuild/tools/modules.py | 10 ++++++++++ test/framework/modulestool.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 14891847cb..6df684db77 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1419,6 +1419,16 @@ def get_setenv_value_from_modulefile(self, mod_name, var_name): return value + def update(self): + """Update after new modules were added.""" + + version = LooseVersion(self.version) + if build_option('update_modules_tool_cache') and version >= LooseVersion('5.3.0'): + out = self.run_module('cachebuild', return_stderr=True, check_output=False) + + if self.testing: + return out + class Lmod(ModulesTool): """Interface to Lmod.""" diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 7b1202c2eb..c9396cf5cf 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -209,6 +209,29 @@ def test_environment_modules_specific(self): mt = EnvironmentModules(testing=True) self.assertIsInstance(mt.loaded_modules(), list) # dummy usage + # test updating module cache + test_modulepath = os.path.join(self.test_installpath, 'modules', 'all') + os.environ['MODULEPATH'] = test_modulepath + test_module_dir = os.path.join(test_modulepath, 'test') + test_module_file = os.path.join(test_module_dir, '1.2.3') + write_file(test_module_file, '#%Module') + build_options = { + 'update_modules_tool_cache': True, + } + init_config(build_options=build_options) + mt = EnvironmentModules(testing=True) + out = mt.update() + os.remove(test_module_file) + os.rmdir(test_module_dir) + + # test cache file has been created if module tool supports it + if LooseVersion(mt.version) >= LooseVersion('5.3.0'): + cache_fp = os.path.join(test_modulepath, '.modulecache') + expected = "Creating %s\n" % cache_fp + self.assertEqual(expected, out, "Module cache created") + self.assertTrue(os.path.exists(cache_fp)) + os.remove(cache_fp) + # initialize Environment Modules tool with non-official version number # pass (fake) full path to 'modulecmd.tcl' via $MODULES_CMD fake_path = os.path.join(self.test_installpath, 'libexec', 'modulecmd.tcl') From d0c4b0ace485ca0ef3b168275e3d7b83f1f47b82 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Thu, 2 Nov 2023 21:14:08 +0100 Subject: [PATCH 408/430] Remove Lmod version check for check_group As minimal supported version of Lmod is now 8.0.0, there is no more need to check Lmod version to verify if inserting check group code in Lua modulefile is supported. --- easybuild/tools/module_generator.py | 19 ++++---------- test/framework/toy_build.py | 39 ++++++++++++----------------- 2 files changed, 21 insertions(+), 37 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 9b1ee024c4..7967b5bda2 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1186,21 +1186,12 @@ def check_group(self, group, error_msg=None): :param group: string with the group name :param error_msg: error message to print for users outside that group """ - lmod_version = self.modules_tool.version - min_lmod_version = '6.0.8' + if error_msg is None: + error_msg = "You are not part of '%s' group of users that have access to this software; " % group + error_msg += "Please consult with user support how to become a member of this group" - if LooseVersion(lmod_version) >= LooseVersion(min_lmod_version): - if error_msg is None: - error_msg = "You are not part of '%s' group of users that have access to this software; " % group - error_msg += "Please consult with user support how to become a member of this group" - - error_msg = 'LmodError("' + error_msg + '")' - res = self.conditional_statement('userInGroup("%s")' % group, error_msg, negative=True) - else: - warn_msg = "Can't generate robust check in Lua modules for users belonging to group %s. " - warn_msg += "Lmod version not recent enough (%s), should be >= %s" - self.log.warning(warn_msg, group, lmod_version, min_lmod_version) - res = '' + error_msg = 'LmodError("' + error_msg + '")' + res = self.conditional_statement('userInGroup("%s")' % group, error_msg, negative=True) return res diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 97e154b23f..35f3771ecf 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -779,30 +779,23 @@ def test_toy_group_check(self): self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt)) elif get_module_syntax() == 'Lua': - lmod_version = os.getenv('LMOD_VERSION', 'NOT_FOUND') - if LooseVersion(lmod_version) >= LooseVersion('6.0.8'): - toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0.lua') - toy_mod_txt = read_file(toy_mod) - - if isinstance(group, tuple): - group_name = group[0] - error_msg_pattern = "Hey, you're not in the '%s' group!" % group_name - else: - group_name = group - error_msg_pattern = "You are not part of '%s' group of users" % group_name - - pattern = '\n'.join([ - r'^if not \( userInGroup\("%s"\) \) then' % group_name, - r' LmodError\("%s[^"]*"\)' % error_msg_pattern, - r'end$', - ]) - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_mod_txt)) + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0.lua') + toy_mod_txt = read_file(toy_mod) + + if isinstance(group, tuple): + group_name = group[0] + error_msg_pattern = "Hey, you're not in the '%s' group!" % group_name else: - pattern = r"Can't generate robust check in Lua modules for users belonging to group %s. " - pattern += r"Lmod version not recent enough \(%s\), should be >= 6.0.8" % lmod_version - regex = re.compile(pattern % group_name, re.M) - self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt)) + group_name = group + error_msg_pattern = "You are not part of '%s' group of users" % group_name + + pattern = '\n'.join([ + r'^if not \( userInGroup\("%s"\) \) then' % group_name, + r' LmodError\("%s[^"]*"\)' % error_msg_pattern, + r'end$', + ]) + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_mod_txt)) else: self.fail("Unknown module syntax: %s" % get_module_syntax()) From 4656a85afb09f4cefb049ff70f81609d55ff6600 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Fri, 3 Nov 2023 09:02:54 +0100 Subject: [PATCH 409/430] Add check_group support for Tcl modulefile "module-info usergroups" modulefile command has been introduced on Environment Modules version 4.6.0. This commit adds support for check_group in Tcl modulefile when required module tool and version is used. --- easybuild/tools/module_generator.py | 14 ++++++++++++-- easybuild/tools/modules.py | 5 +++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 7967b5bda2..86da325c58 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -757,8 +757,18 @@ def check_group(self, group, error_msg=None): :param group: string with the group name :param error_msg: error message to print for users outside that group """ - self.log.warning("Can't generate robust check in TCL modules for users belonging to group %s.", group) - return '' + if self.modules_tool.supports_tcl_check_group: + if error_msg is None: + error_msg = "You are not part of '%s' group of users that have access to this software; " % group + error_msg += "Please consult with user support how to become a member of this group" + + error_msg = 'error "%s"' % error_msg + res = self.conditional_statement('module-info usergroups %s' % group, error_msg, negative=True) + else: + self.log.warning("Can't generate robust check in Tcl modules for users belonging to group %s.", group) + res = '' + + return res def comment(self, msg): """Return string containing given message as a comment.""" diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 14891847cb..f4d84eaadf 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -150,6 +150,8 @@ class ModulesTool(object): REQ_VERSION = None # minimal required version to use getenv Tcl modulefile command REQ_VERSION_TCL_GETENV = None + # minimal required version to check user's group in modulefile + REQ_VERSION_TCL_CHECK_GROUP = None # deprecated version limit (support for versions below this version is deprecated) DEPR_VERSION = None # maximum version allowed (cannot include -beta or rc) @@ -212,6 +214,7 @@ def __init__(self, mod_paths=None, testing=False): self.set_and_check_version() self.supports_depends_on = False self.supports_tcl_getenv = False + self.supports_tcl_check_group = False def __str__(self): """String representation of this ModulesTool instance.""" @@ -1321,6 +1324,7 @@ class EnvironmentModules(EnvironmentModulesTcl): REQ_VERSION_TCL_GETENV = '4.2.0' DEPR_VERSION = '4.0.0' # needs to be set as EnvironmentModules inherits from EnvironmentModulesTcl MAX_VERSION = None + REQ_VERSION_TCL_CHECK_GROUP = '4.6.0' VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' SHOW_HIDDEN_OPTION = '--all' @@ -1349,6 +1353,7 @@ def __init__(self, *args, **kwargs): super(EnvironmentModules, self).__init__(*args, **kwargs) version = LooseVersion(self.version) self.supports_tcl_getenv = version >= LooseVersion(self.REQ_VERSION_TCL_GETENV) + self.supports_tcl_check_group = version >= LooseVersion(self.REQ_VERSION_TCL_CHECK_GROUP) def check_module_function(self, allow_mismatch=False, regex=None): """Check whether selected module tool matches 'module' function definition.""" From 8084d941d7b9444103765601cf3def2d8074cb9c Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Fri, 3 Nov 2023 10:11:30 +0100 Subject: [PATCH 410/430] Add check_group tests for module_generator unit --- test/framework/module_generator.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 015ab6520a..fa8e4a3fd3 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1608,6 +1608,28 @@ def test_generated_module_file_swap(self): # one/1.0 module was swapped for one/1.1 self.assertEqual(loaded_mods[-2]['mod_name'], 'one/1.1') + def test_check_group(self): + """Test check_group method.""" + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + if self.modtool.supports_tcl_check_group: + expected = '\n'.join([ + "if { ![ module-info usergroups group_name ] } {", + " error \"mesg\"", + "}", + '', + ]) + self.assertEqual(expected, self.modgen.check_group("group_name", error_msg="mesg")) + else: + self.assertEqual('', self.modgen.check_group("group_name", error_msg="mesg")) + else: + expected = '\n'.join([ + 'if not ( userInGroup("group_name") ) then', + ' LmodError("mesg")', + 'end', + '', + ]) + self.assertEqual(expected, self.modgen.check_group("group_name", error_msg="mesg")) + class TclModuleGeneratorTest(ModuleGeneratorTest): """Test for module_generator module for Tcl syntax.""" From 9311cb82b7f5f3e85fa058882532d7adbff716fc Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Thu, 28 Dec 2023 08:44:21 +0100 Subject: [PATCH 411/430] Adapt test_toy_group_check now Tcl modulefile may support group check --- test/framework/toy_build.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 35f3771ecf..7f507d576f 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -57,7 +57,7 @@ from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, move_file from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl -from easybuild.tools.modules import Lmod +from easybuild.tools.modules import EnvironmentModules, Lmod from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import nub from easybuild.tools.systemtools import get_shared_lib_ext @@ -774,9 +774,29 @@ def test_toy_group_check(self): self.mock_stdout(False) if get_module_syntax() == 'Tcl': - pattern = "Can't generate robust check in TCL modules for users belonging to group %s." % group_name - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt)) + module_version = LooseVersion(self.modtool.version) + if isinstance(self.modtool, EnvironmentModules) and module_version >= LooseVersion('4.6.0'): + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + toy_mod_txt = read_file(toy_mod) + + if isinstance(group, tuple): + group_name = group[0] + error_msg_pattern = "Hey, you're not in the '%s' group!" % group_name + else: + group_name = group + error_msg_pattern = "You are not part of '%s' group of users" % group_name + + pattern = '\n'.join([ + r'^if \{ \!\[ module-info usergroups %s \] \} \{' % group_name, + r' error "%s[^"]*"' % error_msg_pattern, + r'\}$', + ]) + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_mod_txt)) + else: + pattern = "Can't generate robust check in Tcl modules for users belonging to group %s." % group_name + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt)) elif get_module_syntax() == 'Lua': toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0.lua') From f2a56ea206b9d36649b11ffb67e0a8b208ed0f59 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Wed, 1 Nov 2023 16:17:56 +0100 Subject: [PATCH 412/430] No load storm safe guard for Environment Modules Environment Modules v4+ safely handles automatic module load by not reloading already loaded module. Same goes when unloading module: already unloaded dependency will not be evaluated again. As a result, no safe guard test is required and it should even be avoided to get the module dependency correctly tracked. If module dependency declaration is skipped, no relation binds the loaded modules. Thus unloading the dependency will not lead to the auto unload of the dependent module. Without "is-loaded" safe guard, dependency is always declared and loaded environment is kept consistent by auto_handling mechanism [1] A new ModuleTool property named "supports_safe_auto_load" is introduced. When enabled, module load safe guard code is not generated like if depends_on is enabled. "supports_safe_auto_load" is enabled for EnviromentModules 4.2.4+. Prior v4 versions were not handling safely module load of module parent name if already loaded module were not corresponding to the default version. [1] https://modules.readthedocs.io/en/latest/module.html#envvar-MODULES_AUTO_HANDLING --- easybuild/tools/module_generator.py | 9 ++- easybuild/tools/modules.py | 3 + test/framework/module_generator.py | 110 ++++++++++++++++++---------- 3 files changed, 82 insertions(+), 40 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 86da325c58..bd72da8776 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -915,8 +915,13 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_ cond_tmpl = None + # Environment Modules v4+ safely handles automatic module load by not reloading already + # loaded module. No safe guard test is required and it should even be avoided to get the + # module dependency correctly tracked. + safe_auto_load = self.modules_tool.supports_safe_auto_load + if recursive_unload is None: - recursive_unload = build_option('recursive_mod_unload') or depends_on + recursive_unload = build_option('recursive_mod_unload') or depends_on or safe_auto_load if recursive_unload: # wrapping the 'module load' statement with an 'is-loaded or mode == unload' @@ -927,7 +932,7 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_ # see also http://lmod.readthedocs.io/en/latest/210_load_storms.html cond_tmpl = "[ module-info mode remove ] || %s" - if depends_on: + if depends_on or safe_auto_load: if multi_dep_mods and len(multi_dep_mods) > 1: parent_mod_name = os.path.dirname(mod_name) guard = self.is_loaded(multi_dep_mods[1:]) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 94ebad73d0..b4306342c3 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -215,6 +215,7 @@ def __init__(self, mod_paths=None, testing=False): self.supports_depends_on = False self.supports_tcl_getenv = False self.supports_tcl_check_group = False + self.supports_safe_auto_load = False def __str__(self): """String representation of this ModulesTool instance.""" @@ -1325,6 +1326,7 @@ class EnvironmentModules(EnvironmentModulesTcl): DEPR_VERSION = '4.0.0' # needs to be set as EnvironmentModules inherits from EnvironmentModulesTcl MAX_VERSION = None REQ_VERSION_TCL_CHECK_GROUP = '4.6.0' + REQ_VERSION_SAFE_AUTO_LOAD = '4.2.4' VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' SHOW_HIDDEN_OPTION = '--all' @@ -1354,6 +1356,7 @@ def __init__(self, *args, **kwargs): version = LooseVersion(self.version) self.supports_tcl_getenv = version >= LooseVersion(self.REQ_VERSION_TCL_GETENV) self.supports_tcl_check_group = version >= LooseVersion(self.REQ_VERSION_TCL_CHECK_GROUP) + self.supports_safe_auto_load = version >= LooseVersion(self.REQ_VERSION_SAFE_AUTO_LOAD) def check_module_function(self, allow_mismatch=False, regex=None): """Check whether selected module tool matches 'module' function definition.""" diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index fa8e4a3fd3..83fd1a2e0d 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -266,24 +266,32 @@ def test_load(self): """Test load part in generated module file.""" if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: - # default: guarded module load (which implies no recursive unloading) - expected = '\n'.join([ - '', - "if { ![ is-loaded mod_name ] } {", - " module load mod_name", - "}", - '', - ]) + if not self.modtool.supports_safe_auto_load: + # default: guarded module load (which implies no recursive unloading) + expected = '\n'.join([ + '', + "if { ![ is-loaded mod_name ] } {", + " module load mod_name", + "}", + '', + ]) + else: + expected = '\n'.join([ + '', + "module load mod_name", + '', + ]) self.assertEqual(expected, self.modgen.load_module("mod_name")) - # with recursive unloading: no if is-loaded guard - expected = '\n'.join([ - '', - "if { [ module-info mode remove ] || ![ is-loaded mod_name ] } {", - " module load mod_name", - "}", - '', - ]) + if not self.modtool.supports_safe_auto_load: + # with recursive unloading: no if is-loaded guard + expected = '\n'.join([ + '', + "if { [ module-info mode remove ] || ![ is-loaded mod_name ] } {", + " module load mod_name", + "}", + '', + ]) self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload=True)) init_config(build_options={'recursive_mod_unload': True}) @@ -353,13 +361,24 @@ def test_load_multi_deps(self): res = self.modgen.load_module('Python/3.7.4', multi_dep_mods=multi_dep_mods) if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: - expected = '\n'.join([ - '', - "if { ![ is-loaded Python/3.7.4 ] && ![ is-loaded Python/2.7.16 ] } {", - " module load Python/3.7.4", - '}', - '', - ]) + if not self.modtool.supports_safe_auto_load: + expected = '\n'.join([ + '', + "if { ![ is-loaded Python/3.7.4 ] && ![ is-loaded Python/2.7.16 ] } {", + " module load Python/3.7.4", + '}', + '', + ]) + else: + expected = '\n'.join([ + '', + "if { [ module-info mode remove ] || [ is-loaded Python/2.7.16 ] } {", + " module load Python", + '} else {', + " module load Python/3.7.4", + '}', + '', + ]) else: # Lua syntax expected = '\n'.join([ '', @@ -401,14 +420,26 @@ def test_load_multi_deps(self): res = self.modgen.load_module('foo/1.2.3', multi_dep_mods=multi_dep_mods) if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: - expected = '\n'.join([ - '', - "if { ![ is-loaded foo/1.2.3 ] && ![ is-loaded foo/2.3.4 ] && " + - "![ is-loaded foo/3.4.5 ] && ![ is-loaded foo/4.5.6 ] } {", - " module load foo/1.2.3", - '}', - '', - ]) + if not self.modtool.supports_safe_auto_load: + expected = '\n'.join([ + '', + "if { ![ is-loaded foo/1.2.3 ] && ![ is-loaded foo/2.3.4 ] && " + + "![ is-loaded foo/3.4.5 ] && ![ is-loaded foo/4.5.6 ] } {", + " module load foo/1.2.3", + '}', + '', + ]) + else: + expected = '\n'.join([ + '', + "if { [ module-info mode remove ] || [ is-loaded foo/2.3.4 ] || [ is-loaded foo/3.4.5 ] " + + "|| [ is-loaded foo/4.5.6 ] } {", + " module load foo", + "} else {", + " module load foo/1.2.3", + '}', + '', + ]) else: # Lua syntax expected = '\n'.join([ '', @@ -453,13 +484,16 @@ def test_load_multi_deps(self): res = self.modgen.load_module('one/1.0', multi_dep_mods=['one/1.0']) if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: - expected = '\n'.join([ - '', - "if { ![ is-loaded one/1.0 ] } {", - " module load one/1.0", - '}', - '', - ]) + if not self.modtool.supports_safe_auto_load: + expected = '\n'.join([ + '', + "if { ![ is-loaded one/1.0 ] } {", + " module load one/1.0", + '}', + '', + ]) + else: + expected = '\nmodule load one/1.0\n' else: # Lua syntax expected = '\n'.join([ '', From a0fa9e52ef90a73eacbd7b57dce3de19a2b1313b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 28 Aug 2024 01:57:29 +0200 Subject: [PATCH 413/430] Resolve template also for extension version string --- easybuild/framework/easyblock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 72fa56cab8..437724cda4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1748,7 +1748,8 @@ def _make_extension_list(self): if isinstance(ext, str): exts_list.append((resolve_template(ext, self.cfg.template_values), )) else: - exts_list.append((resolve_template(ext[0], self.cfg.template_values), ext[1])) + exts_list.append((resolve_template(ext[0], self.cfg.template_values), + resolve_template(ext[1], self.cfg.template_values))) return exts_list def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True): From 57cc313ac67682ea3411fedfc602107ba52b05d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 28 Aug 2024 18:39:51 +0200 Subject: [PATCH 414/430] Fix accidental module whitespace/order changes --- easybuild/tools/module_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 8b829c75b7..895cbc5cd1 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -829,6 +829,8 @@ def get_description(self, conflict=True): for line in self._generate_whatis_lines() ]) + lines.extend(['', "set root " + self.app.installdir]) + if self.app.cfg['moduleloadnoconflict']: cond_unload = self.conditional_statement(self.is_loaded('%(name)s'), "module unload %(name)s") lines.extend([ @@ -844,9 +846,7 @@ def get_description(self, conflict=True): # - 'conflict Compiler/GCC/4.8.2/OpenMPI' for 'Compiler/GCC/4.8.2/OpenMPI/1.6.4' lines.extend(['', "conflict %s" % os.path.dirname(self.app.short_mod_name)]) - lines.extend(['', "set root " + self.app.installdir]) - - return '\n'.join([''] + lines + ['']) + return '\n'.join(lines + ['']) def getenv_cmd(self, envvar, default=None): """ @@ -1279,7 +1279,7 @@ def get_description(self, conflict=True): # https://github.com/TACC/Lmod/issues/428 lines.extend(['', self.conditional_statement(self.check_version("8", "2", "8"), extensions_stmt)]) - return '\n'.join([''] + lines + ['']) + return '\n'.join(lines + ['']) def getenv_cmd(self, envvar, default=None): """ From ac9ddeaa06c6634268d7327b8e280348221b2eb1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 28 Aug 2024 20:21:57 +0200 Subject: [PATCH 415/430] enhance test_exts_list to verify that template values used in extension name/version are resolved as expected --- test/framework/easyconfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 7dc088bff3..3db9386310 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -522,7 +522,8 @@ def test_exts_list(self): with self.mocked_stdout_stderr(): modfile = os.path.join(eb.make_module_step(), 'PI', '3.14' + eb.module_generator.MODULE_FILE_EXTENSION) modtxt = read_file(modfile) - regex = re.compile('EBEXTSLISTPI.*ext1-1.0,ext2-2.0') + # verify that templates used for extensions are resolved as they should + regex = re.compile('EBEXTSLISTPI.*"ext1-1.0,ext2-2.0,ext-PI-3.14,ext-pi-3.0') self.assertTrue(regex.search(modtxt), "Pattern '%s' found in: %s" % (regex.pattern, modtxt)) def test_extensions_templates(self): From b3fefb954c58262ac5f465e1eac107ce86f1d660 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 30 Aug 2024 19:04:11 +0200 Subject: [PATCH 416/430] restore %(software_commit)s in TEMPLATE_NAMES_DYNAMIC --- easybuild/framework/easyconfig/templates.py | 8 +++++--- test/framework/options.py | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 57ce5d061a..ef0035d76d 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -89,9 +89,6 @@ # template values which are only generated dynamically TEMPLATE_NAMES_DYNAMIC = { 'arch': 'System architecture (e.g. x86_64, aarch64, ppc64le, ...)', - 'sysroot': "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" - "as specify by the --sysroot configuration option", - 'mpi_cmd_prefix': 'Prefix command for running MPI programs (with default number of ranks)', 'cuda_compute_capabilities': "Comma-separated list of CUDA compute capabilities, as specified via " "--cuda-compute-capabilities configuration option or " "via cuda_compute_capabilities easyconfig parameter", @@ -105,6 +102,11 @@ 'cuda_int_semicolon_sep': 'Semicolon-separated list of integer CUDA compute capabilities', 'cuda_sm_comma_sep': 'Comma-separated list of sm_* values that correspond with CUDA compute capabilities', 'cuda_sm_space_sep': 'Space-separated list of sm_* values that correspond with CUDA compute capabilities', + 'mpi_cmd_prefix': 'Prefix command for running MPI programs (with default number of ranks)', + 'software_commit': "Git commit id to use for the software as specified by --software-commit command line option", + 'sysroot': "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" + "as specify by the --sysroot configuration option", + } # constant templates that can be used in easyconfigs diff --git a/test/framework/options.py b/test/framework/options.py index 5739527341..089b068e62 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -604,6 +604,8 @@ def run_test(fmt=None): r'^``%\(arch\)s``\s+System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)\s*$', r'^``%\(cuda_cc_space_sep\)s``\s+Space-separated list of CUDA compute capabilities\s*$', r'^``SOURCE_TAR_GZ``\s+Source \.tar\.gz bundle\s+``%\(name\)s-%\(version\)s.tar.gz``\s*$', + r'^``%\(software_commit\)s``\s+Git commit id to use for the software as specified ' + 'by --software-commit command line option', ] else: pattern_lines = [ @@ -616,6 +618,8 @@ def run_test(fmt=None): r'^\s+%\(arch\)s: System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)$', r'^\s+%\(cuda_cc_space_sep\)s: Space-separated list of CUDA compute capabilities$', r'^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)$', + r'^\s+%\(software_commit\)s: Git commit id to use for the software as specified ' + 'by --software-commit command line option', ] for pattern_line in pattern_lines: From 9ca954f7dae9a0166ad51e03242c54951666d536 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 13:38:21 +0100 Subject: [PATCH 417/430] Run unit tests on an updated version of Modules 4 Environment Modules was updated on EL8 from version 4.1.4 to version 4.5.2+patches which can be assimilated to version 4.5.3. Seems interesting to update the modules4 version used in unit_tests CI workflow to match Environment Modules version found in EL8. --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5bf29eb164..9441dca0e2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-20.04 outputs: lmod8: Lmod-8.7.6 - modules4: modules-4.1.4 + modules4: modules-4.5.3 steps: - run: "true" build: From ef619509ca3e65742550a0310f811c47f144d9be Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Wed, 27 Dec 2023 20:17:08 +0100 Subject: [PATCH 418/430] Fix test_cray_reset/test_independence tests for EnvironmentModules test_cray_reset and test_independence tests successively load all Cray* modules. One Cray* module does not unload other Cray* modules loaded which give an inconsistent environment. Environment Modules tool version 4.2 and above does not allow setting up an inconsistent environment without --force option set. To fix test_cray_reset and test_independence tests for Environment Modules tool, environment is purged between each toolchain setup. --- test/framework/toolchain.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index df0bb53ebc..19df6f281d 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -50,6 +50,7 @@ from easybuild.tools.environment import setvar from easybuild.tools.filetools import adjust_permissions, copy_dir, find_eb_script, mkdir from easybuild.tools.filetools import read_file, symlink, write_file, which +from easybuild.tools.modules import EnvironmentModules from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template @@ -470,6 +471,10 @@ def test_cray_reset(self): init_config(build_options={'optarch': 'test', 'silent': True}) for tcname in ['CrayGNU', 'CrayCCE', 'CrayIntel']: + # Cray* modules do not unload other Cray* modules thus loading a second Cray* module + # makes environment inconsistent which is not allowed by Environment Modules tool + if isinstance(self.modtool, EnvironmentModules): + self.modtool.purge() tc = self.get_toolchain(tcname, version='2015.06-XC') tc.set_options({'dynamic': True}) with self.mocked_stdout_stderr(): @@ -2208,6 +2213,10 @@ def test_independence(self): # purposely obtain toolchains several times in a row, value for $CFLAGS should not change for _ in range(3): for tcname, tcversion in toolchains: + # Cray* modules do not unload other Cray* modules thus loading a second Cray* module + # makes environment inconsistent which is not allowed by Environment Modules tool + if isinstance(self.modtool, EnvironmentModules): + self.modtool.purge() tc = get_toolchain({'name': tcname, 'version': tcversion}, {}, mns=ActiveMNS(), modtool=self.modtool) # also check whether correct compiler flag for OpenMP is used while we're at it From 3b69e463242138d5969c03aa34df74980c4fec89 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 1 Sep 2024 08:16:52 +0200 Subject: [PATCH 419/430] Run unit tests on Environment Modules 5 It seems interesting to also test Modules 5, especially that a lot of changes are introduced compared to Modules 4. Some of these features are now supported by EnvironmentModules framework's class. Use Environment Modules 5.3 as this is version currently shipped on EL9 systems. --- .github/workflows/unit_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 9441dca0e2..0bcbf3a466 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -15,6 +15,7 @@ jobs: outputs: lmod8: Lmod-8.7.6 modules4: modules-4.5.3 + modules5: modules-5.3.1 steps: - run: "true" build: @@ -28,6 +29,7 @@ jobs: # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context - ${{needs.setup.outputs.lmod8}} - ${{needs.setup.outputs.modules4}} + - ${{needs.setup.outputs.modules5}} lc_all: [""] include: # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) @@ -158,7 +160,7 @@ jobs: export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH eb --version # tell EasyBuild which modules tool is available - if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then + if [[ ${{matrix.modules_tool}} =~ ^modules- ]]; then export EASYBUILD_MODULES_TOOL=EnvironmentModules else export EASYBUILD_MODULES_TOOL=Lmod From 1cfbfa37ffb6f82913d549ffc774013f4b53f2b8 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Thu, 28 Dec 2023 11:57:01 +0100 Subject: [PATCH 420/430] Adapt test_toy_toy test to produce 2 different module names "test_toy_toy" test was generating two versions of "toy" module. Both modules expressed a reflexive conflict (against "toy") and toy/0.0-two loaded toy/0.0-one. As these two modules shared the same name ("toy") and due to the reflexive conflict they express, environment obtained when loading toy/0.0-two is inconsistent as it tries to load toy/0.0-one. Environment Modules version 4.2 (and above) detects this inconsistency and raise an error unless --force option is set. "test_toy_toy" test is adapted to build a "toy2" module for "toy2" easyconfig. --- test/framework/toy_build.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 3c70ceda5a..4288cb7e0e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2330,6 +2330,7 @@ def test_reproducibility_ext_easyblocks(self): def test_toy_toy(self): """Test building two easyconfigs in a single go, with one depending on the other.""" + topdir = os.path.dirname(os.path.abspath(__file__)) toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') toy_ec_txt = read_file(toy_ec_file) @@ -2341,9 +2342,17 @@ def test_toy_toy(self): ]) write_file(ec1, ec1_txt) + # adapt toy easyconfig for toy2 to produce a modulefile with a dedicated + # name ('toy2' instead of 'toy') ec2 = os.path.join(self.test_prefix, 'toy2.eb') ec2_txt = '\n'.join([ toy_ec_txt, + "name = 'toy2'", + "easyblock = 'EB_toy'", + "sources = ['toy/toy-0.0.tar.gz']", + "patches = []", + "sanity_check_paths = { 'files': ['bin/toy2'], 'dirs': ['bin']}", + "prebuildopts = 'mv toy.source toy2.c &&'", "versionsuffix = '-two'", "dependencies = [('toy', '0.0', '-one')]", ]) @@ -2353,7 +2362,7 @@ def test_toy_toy(self): self._test_toy_build(ec_file=self.test_prefix, verify=False) mod1 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-one') - mod2 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-two') + mod2 = os.path.join(self.test_installpath, 'modules', 'all', 'toy2', '0.0-two') if get_module_syntax() == 'Lua': mod1 += '.lua' mod2 += '.lua' @@ -2366,15 +2375,15 @@ def test_toy_toy(self): self.assertTrue(load1_regex.search(mod2_txt), "Pattern '%s' found in: %s" % (load1_regex.pattern, mod2_txt)) # Check the contents of the dumped env in the reprod dir to ensure it contains the dependency load - reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-two', 'easybuild', 'reprod') - dumpenv_script = os.path.join(reprod_dir, 'toy-0.0-two.env') + reprod_dir = os.path.join(self.test_installpath, 'software', 'toy2', '0.0-two', 'easybuild', 'reprod') + dumpenv_script = os.path.join(reprod_dir, 'toy2-0.0-two.env') reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) self.assertExists(reprod_dumpenv) # Check contents of the dumpenv script patterns = [ """#!/bin/bash""", - """# usage: source toy-0.0-two.env""", + """# usage: source toy2-0.0-two.env""", # defining build env """module load toy/0.0-one""", """# (no build environment defined)""", From f40f2086cb57b53cb43abe88e78549d0134c15ef Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 1 Sep 2024 11:51:06 +0200 Subject: [PATCH 421/430] Take safe_auto_load into account on easyblock/easyconfig/toy_build tests --- test/framework/easyblock.py | 53 ++++++++++++++++++++++-------------- test/framework/easyconfig.py | 33 ++++++++++++++++------ test/framework/toy_build.py | 21 ++++++++++---- 3 files changed, 72 insertions(+), 35 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 01014e3ee1..18cef86fc9 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -754,21 +754,26 @@ def test_make_module_dep(self): eb.prepare_step() if get_module_syntax() == 'Tcl': - tc_load = '\n'.join([ - "if { ![ is-loaded gompi/2018a ] } {", - " module load gompi/2018a", - "}", - ]) - fftw_load = '\n'.join([ - "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {", - " module load FFTW/3.3.7-gompi-2018a", - "}", - ]) - lapack_load = '\n'.join([ - "if { ![ is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28 ] } {", - " module load OpenBLAS/0.2.20-GCC-6.4.0-2.28", - "}", - ]) + if self.modtool.supports_safe_auto_load: + tc_load = "module load gompi/2018a" + fftw_load = "module load FFTW/3.3.7-gompi-2018a" + lapack_load = "module load OpenBLAS/0.2.20-GCC-6.4.0-2.28" + else: + tc_load = '\n'.join([ + "if { ![ is-loaded gompi/2018a ] } {", + " module load gompi/2018a", + "}", + ]) + fftw_load = '\n'.join([ + "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {", + " module load FFTW/3.3.7-gompi-2018a", + "}", + ]) + lapack_load = '\n'.join([ + "if { ![ is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28 ] } {", + " module load OpenBLAS/0.2.20-GCC-6.4.0-2.28", + "}", + ]) elif get_module_syntax() == 'Lua': tc_load = '\n'.join([ 'if not ( isloaded("gompi/2018a") ) then', @@ -798,12 +803,18 @@ def test_make_module_dep(self): } if get_module_syntax() == 'Tcl': - fftw_load = '\n'.join([ - "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {", - " module unload FFTW", - " module load FFTW/3.3.7-gompi-2018a", - "}", - ]) + if self.modtool.supports_safe_auto_load: + fftw_load = '\n'.join([ + "module unload FFTW", + "module load FFTW/3.3.7-gompi-2018a", + ]) + else: + fftw_load = '\n'.join([ + "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {", + " module unload FFTW", + " module load FFTW/3.3.7-gompi-2018a", + "}", + ]) elif get_module_syntax() == 'Lua': fftw_load = '\n'.join([ 'if not ( isloaded("FFTW/3.3.7-gompi-2018a") ) then', diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a780fcba6b..ef71cf9514 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4792,7 +4792,10 @@ def test_recursive_module_unload(self): recursive_unload_pat = r'if mode\(\) == "unload" or not \( isloaded\("%(mod)s"\) \) then\n' recursive_unload_pat += r'\s*load\("%(mod)s"\)' else: - guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s' + if self.modtool.supports_safe_auto_load: + guarded_load_pat = r'\nmodule load %(mod)s' + else: + guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s' recursive_unload_pat = r'if { \[ module-info mode remove \] \|\| \!\[ is-loaded %(mod)s \] } {\n' recursive_unload_pat += r'\s*module load %(mod)s' @@ -4831,10 +4834,16 @@ def test_recursive_module_unload(self): eb_bis.prepare_step() eb_bis.make_module_step() modtxt = read_file(test_module) - fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt) - self.assertFalse(guarded_load_regex.search(modtxt), fail_msg) - fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt) - self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg) + if self.modtool.supports_safe_auto_load: + fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertTrue(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg) + else: + fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertFalse(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg) # recursive_mod_unload build option is honored update_build_option('recursive_mod_unload', True) @@ -4844,10 +4853,16 @@ def test_recursive_module_unload(self): eb.prepare_step() eb.make_module_step() modtxt = read_file(test_module) - fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt) - self.assertFalse(guarded_load_regex.search(modtxt), fail_msg) - fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt) - self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg) + if self.modtool.supports_safe_auto_load: + fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertTrue(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg) + else: + fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertFalse(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg) # disabling via easyconfig parameter works even when recursive_mod_unload build option is enabled self.assertTrue(build_option('recursive_mod_unload')) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 4288cb7e0e..38a0595b6c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3209,11 +3209,22 @@ def test_toy_multi_deps(self): 'end', ]) else: - expected = '\n'.join([ - 'if { ![ is-loaded GCC/4.6.3 ] && ![ is-loaded GCC/7.3.0-2.30 ] } {', - ' module load GCC/4.6.3', - '}', - ]) + if not self.modtool.supports_safe_auto_load: + expected = '\n'.join([ + 'if { ![ is-loaded GCC/4.6.3 ] && ![ is-loaded GCC/7.3.0-2.30 ] } {', + ' module load GCC/4.6.3', + '}', + ]) + else: + expected = '\n'.join([ + '', + "if { [ module-info mode remove ] || [ is-loaded GCC/7.3.0-2.30 ] } {", + " module load GCC", + '} else {', + " module load GCC/4.6.3", + '}', + '', + ]) self.assertIn(expected, toy_mod_txt) From 34ef6d5ed8a7f9fcedf712030cfbfeebdf6665a0 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 1 Sep 2024 14:03:08 +0200 Subject: [PATCH 422/430] Take Tcl getenv into account on easyblock tests --- test/framework/easyblock.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 18cef86fc9..e7218b2273 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -284,8 +284,11 @@ def test_make_module_extend_modpath(self): txt = eb.make_module_extend_modpath() if module_syntax == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } ' - home += r'else { concat "HOME_NOT_DEFINED" } \]' + if self.modtool.supports_tcl_getenv: + home = r'\[getenv HOME "HOME_NOT_DEFINED"\]' + else: + home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } ' + home += r'else { concat "HOME_NOT_DEFINED" } \]' fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -327,9 +330,12 @@ def test_make_module_extend_modpath(self): for envvar in list_of_envvars: if module_syntax == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar - module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar - module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED') + if self.modtool.supports_tcl_getenv: + module_envvar = r'\[getenv %s "%s"]' % (envvar, envvar + '_NOT_DEFINED') + else: + module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar + module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar + module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED') fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded From 29bcd1c0b122d15b6e148e4af04dd50ab87362df Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 30 Aug 2024 21:26:33 +0000 Subject: [PATCH 423/430] Add deprecation warning for /etc/easybuild.d If you don't set XDG_CONFIG_DIRS and files are present in /etc/easybuild.d we now get WARNING: Deprecated functionality, will no longer work in EasyBuild v6.0: Using /etc/easybuild.d is deprecated. Please use /etc/xdg/easybuild.d instead or add /etc to XDG_CONFIG_DIRS; see https://docs.easybuild.io/deprecated-functionality/ for more information This addresses https://github.com/easybuilders/easybuild-framework/pull/4591#issuecomment-2310276909 --- easybuild/tools/options.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 935f450782..4ff6300a26 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -213,6 +213,12 @@ class EasyBuildOptions(GeneralOption): DEFAULT_LOGLEVEL = 'INFO' DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES[:] + if 'XDG_CONFIG_DIRS' not in os.environ: + old_etc_location = os.path.join('/etc', 'easybuild.d') + if os.path.isdir(old_etc_location) and glob.glob(os.path.join(old_etc_location, '*.cfg')): + _log.deprecated(f"Using {old_etc_location} is deprecated. Please use /etc/xdg/easybuild.d " + "instead or add /etc to XDG_CONFIG_DIRS", '6.0') + if os.path.exists(DEFAULT_USER_CFGFILE): DEFAULT_CONFIGFILES.append(DEFAULT_USER_CFGFILE) From b8103e103211266f7497c3cba413012ec50acd4a Mon Sep 17 00:00:00 2001 From: Patrice Peterson Date: Wed, 4 Sep 2024 14:34:56 +0200 Subject: [PATCH 424/430] Appease hound --- easybuild/tools/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4ff6300a26..a79c458a42 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -216,8 +216,8 @@ class EasyBuildOptions(GeneralOption): if 'XDG_CONFIG_DIRS' not in os.environ: old_etc_location = os.path.join('/etc', 'easybuild.d') if os.path.isdir(old_etc_location) and glob.glob(os.path.join(old_etc_location, '*.cfg')): - _log.deprecated(f"Using {old_etc_location} is deprecated. Please use /etc/xdg/easybuild.d " - "instead or add /etc to XDG_CONFIG_DIRS", '6.0') + _log.deprecated(f"Using {old_etc_location} is deprecated. Please use " + "/etc/xdg/easybuild.d instead or add /etc to XDG_CONFIG_DIRS", '6.0') if os.path.exists(DEFAULT_USER_CFGFILE): DEFAULT_CONFIGFILES.append(DEFAULT_USER_CFGFILE) From 0d407089ec752f47bb4b80bb6133598241dc77b0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Sep 2024 17:13:53 +0200 Subject: [PATCH 425/430] verify checksums with `--fetch` After `--fetch` finishes it is reasonable to expect that the build won't fail due to missing or wrong sources. However putting the checksum-step into the source-step skips over the verification and one would need to use `--stop=source` which does a lot more than necessary, e.g. creating build dirs and environment variables. Move the checksums step into the fetch step and add a test for that. --- easybuild/framework/easyblock.py | 16 ++++------------ test/framework/options.py | 13 ++++++++++++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d119a299b7..71c69e1821 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4038,15 +4038,6 @@ def ready_step_spec(initial): return get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False, initial=initial) - source_substeps = [ - (False, lambda x: x.checksum_step), - (True, lambda x: x.extract_step), - ] - - def source_step_spec(initial): - """Return source step specified.""" - return get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial) - install_substeps = [ (False, lambda x: x.stage_install_step), (False, lambda x: x.make_installdir), @@ -4060,6 +4051,7 @@ def install_step_spec(initial): # format for step specifications: (step_name, description, list of functions, skippable) # core steps that are part of the iterated loop + extract_step_spec = (SOURCE_STEP, "unpacking", [lambda x: x.extract_step], True) patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step], True) prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False) configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True) @@ -4069,9 +4061,9 @@ def install_step_spec(initial): # part 1: pre-iteration + first iteration steps_part1 = [ - (FETCH_STEP, 'fetching files', [lambda x: x.fetch_step], False), + (FETCH_STEP, 'fetching files', [lambda x: x.fetch_step, lambda x: x.checksum_step], False), ready_step_spec(True), - source_step_spec(True), + extract_step_spec, patch_step_spec, prepare_step_spec, configure_step_spec, @@ -4085,7 +4077,7 @@ def install_step_spec(initial): # not all parts of all steps need to be rerun (see e.g., ready, prepare) steps_part2 = [ ready_step_spec(False), - source_step_spec(False), + extract_step_spec, patch_step_spec, prepare_step_spec, configure_step_spec, diff --git a/test/framework/options.py b/test/framework/options.py index 089b068e62..0c7ae93f25 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5497,7 +5497,7 @@ def test_fetch(self): # which might trip up the dependency resolution (see #4298) for ec in ('toy-0.0.eb', 'toy-0.0-deps.eb'): args = [ec, '--fetch'] - stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) + stdout, _ = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) patterns = [ r"^== fetching files\.\.\.$", @@ -5510,6 +5510,17 @@ def test_fetch(self): regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$") self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + # --fetch should also verify the checksums + tmpdir = tempfile.mkdtemp(prefix='easybuild-sources') + write_file(os.path.join(tmpdir, 'toy-0.0.tar.gz'), 'Make checksum check fail') + args = ['--sourcepath=%s:%s' % (tmpdir, self.test_sourcepath), '--fetch', 'toy-0.0.eb'] + with self.mocked_stdout_stderr(): + pattern = 'Checksum verification for .*/toy-0.0.tar.gz .*failed' + self.assertErrorRegex(EasyBuildError, pattern, self.eb_main, args, do_build=True, raise_error=True) + # We can avoid that failure by ignoring the checksums + args.append('--ignore-checksums') + self.eb_main(args, do_build=True, raise_error=True) + def test_parse_external_modules_metadata(self): """Test parse_external_modules_metadata function.""" # by default, provided external module metadata cfg files are picked up From 0162664461b77252ccfa852c951fa11a8cf1e67f Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sat, 31 Aug 2024 15:29:10 +0200 Subject: [PATCH 426/430] Consistent naming of 'Environment Modules' Use same name and character case to refer to 'Environment Modules' tool. --- easybuild/tools/config.py | 2 +- easybuild/tools/module_generator.py | 2 +- easybuild/tools/modules.py | 16 ++++++++-------- test/framework/module_generator.py | 2 +- test/framework/modules.py | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 8eb7d0db6b..b4727abb09 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -735,7 +735,7 @@ def container_path(): def get_modules_tool(): """ - Return modules tool (EnvironmentModulesC, Lmod, ...) + Return modules tool (EnvironmentModules, Lmod, ...) """ # 'modules_tool' key will only be present if EasyBuild config is initialized return ConfigurationVariables().get('modules_tool', None) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index ab03bc2d04..f5ea7fb46c 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1018,7 +1018,7 @@ def set_alias(self, key, value): def set_as_default(self, module_dir_path, module_version, mod_symlink_paths=None): """ - Create a .version file inside the package module folder in order to set the default version for TMod + Create a .version file inside the package module folder in order to set the default version :param module_dir_path: module directory path, e.g. $HOME/easybuild/modules/all/Bison :param module_version: module version, e.g. 3.0.4 diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index b4306342c3..992527f42c 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -745,7 +745,7 @@ def modulefile_path(self, mod_name, strip_ext=False): :param mod_name: module name :param strip_ext: strip (.lua) extension from module fileame (if present)""" # (possible relative) path is always followed by a ':', and may be prepended by whitespace - # this works for both environment modules and Lmod + # this works for both Environment Modules and Lmod modpath_re = re.compile(r'^\s*(?P[^/\n]*/[^\s]+):$', re.M) modpath = self.get_value_from_modulefile(mod_name, modpath_re) @@ -1180,7 +1180,7 @@ def update(self): class EnvironmentModulesC(ModulesTool): - """Interface to (C) environment modules (modulecmd).""" + """Interface to (C) Environment Modules (modulecmd).""" NAME = "Environment Modules" COMMAND = "modulecmd" REQ_VERSION = '3.2.10' @@ -1195,7 +1195,7 @@ def run_module(self, *args, **kwargs): if isinstance(args[0], (list, tuple,)): args = args[0] - # some versions of Cray's environment modules tool (3.2.10.x) include a "source */init/bash" command + # some versions of Cray's Environment Modules tool (3.2.10.x) include a "source */init/bash" command # in the output of some "modulecmd python load" calls, which is not a valid Python command, # which must be stripped out to avoid "invalid syntax" errors when evaluating the output def tweak_stdout(txt): @@ -1237,9 +1237,9 @@ def get_setenv_value_from_modulefile(self, mod_name, var_name): class EnvironmentModulesTcl(EnvironmentModulesC): - """Interface to (Tcl) environment modules (modulecmd.tcl).""" + """Interface to (ancient Tcl-only) Environment Modules (modulecmd.tcl).""" NAME = "ancient Tcl-only Environment Modules" - # Tcl environment modules have no --terse (yet), + # ancient Tcl-only Environment Modules have no --terse (yet), # -t must be added after the command ('avail', 'list', etc.) TERSE_OPTION = (1, '-t') COMMAND = 'modulecmd.tcl' @@ -1253,7 +1253,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC): def set_path_env_var(self, key, paths): """Set environment variable with given name to the given list of paths.""" super(EnvironmentModulesTcl, self).set_path_env_var(key, paths) - # for Tcl environment modules, we need to make sure the _modshare env var is kept in sync + # for Tcl Environment Modules, we need to make sure the _modshare env var is kept in sync setvar('%s_modshare' % key, ':1:'.join(paths), verbose=False) def run_module(self, *args, **kwargs): @@ -1317,7 +1317,7 @@ def remove_module_path(self, path, set_mod_paths=True): class EnvironmentModules(EnvironmentModulesTcl): - """Interface to environment modules 4.0+""" + """Interface to Environment Modules 4.0+""" NAME = "Environment Modules" COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') COMMAND_ENVIRONMENT = 'MODULES_CMD' @@ -1786,7 +1786,7 @@ def avail_modules_tools(): def modules_tool(mod_paths=None, testing=False): """ - Return interface to modules tool (environment modules (C, Tcl), or Lmod) + Return interface to modules tool (EnvironmentModules, Lmod, ...) """ # get_modules_tool might return none (e.g. if config was not initialized yet) modules_tool = get_modules_tool() diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 83fd1a2e0d..c122c810be 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -684,7 +684,7 @@ def test_swap(self): # create tiny test Tcl module to make sure that tested modules tools support single-argument swap # see https://github.com/easybuilders/easybuild-framework/issues/3396; - # this is known to fail with the ancient Tcl-only implementation of environment modules, + # this is known to fail with the ancient Tcl-only implementation of Environment Modules, # but that's considered to be a non-issue (since this is mostly relevant for Cray systems, # which are either using EnvironmentModulesC (3.2.10), EnvironmentModules (4.x) or Lmod... if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl and self.modtool.__class__ != EnvironmentModulesTcl: diff --git a/test/framework/modules.py b/test/framework/modules.py index 03e217374c..e8361c506c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -460,7 +460,7 @@ def test_load(self): # if GCC is loaded again, $EBROOTGCC should be set again, and GCC should be listed last self.modtool.load(['GCC/6.4.0-2.28']) - # environment modules v4+ does not reload already loaded modules + # Environment Modules v4+ does not reload already loaded modules if not isinstance(self.modtool, EnvironmentModules): self.assertTrue(os.environ.get('EBROOTGCC')) @@ -1413,7 +1413,7 @@ def test_exit_code_check(self): if isinstance(self.modtool, Lmod): error_pattern = "Module command '.*load nosuchmoduleavailableanywhere' failed with exit code" else: - # Tcl implementations exit with 0 even when a non-existing module is loaded... + # Environment Modules exits with 0 even when a non-existing module is loaded... error_pattern = "Unable to locate a modulefile for 'nosuchmoduleavailableanywhere'" self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.load, ['nosuchmoduleavailableanywhere']) From 928f7459e8585cab476a7dbd87924e7ef9cbcb64 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sat, 31 Aug 2024 15:31:31 +0200 Subject: [PATCH 427/430] No module-version in show command output on EnvironmentModules Unlike EnvironmentModulesC and EnvironmentModulesTcl, EnvironmentModules does not produce "module-*" commands in the output of a "module show" command. EnvironmentModules still produces a WARNING if a symbolic version is defined multiple times. Existing test of this behavior is updated to display newer warning message. --- easybuild/tools/module_generator.py | 2 +- easybuild/tools/modules.py | 3 ++- test/framework/modules.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index f5ea7fb46c..a05d9b8941 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -346,7 +346,7 @@ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None): module_version_statement = "module-version %(modname)s %(sym_version)s" - # for Environment Modules we need to guard the module-version statement, + # for EnvironmentModulesC we need to guard the module-version statement, # to avoid "Duplicate version symbol" warning messages where EasyBuild trips over, # which occur because the .modulerc is parsed twice # "module-info version " returns its argument if that argument is not a symbolic version (yet), diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 992527f42c..086a56f669 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -590,7 +590,8 @@ def mod_exists_via_show(mod_name): self.log.debug("Skipping warning line '%s'", line) continue - # skip lines that start with 'module-' (like 'module-version'), + # skip lines that start with 'module-' (like 'module-version') + # that may appear with EnvironmentModulesC or EnvironmentModulesTcl, # see https://github.com/easybuilders/easybuild-framework/issues/3376 if line.startswith('module-'): self.log.debug("Skipping line '%s' since it starts with 'module-'", line) diff --git a/test/framework/modules.py b/test/framework/modules.py index e8361c506c..5a2026258f 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -343,12 +343,12 @@ def test_exist(self): easybuild.tools.modules.MODULE_SHOW_CACHE.clear() self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) - # mimic more verbose stderr output produced by old Tmod version, - # including a warning produced when multiple .modulerc files are being picked up + # mimic "module-*" output produced by EnvironmentModulesC or EnvironmentModulesTcl + # mimic warning produced by Environment Modules when a symbol is defined multiple times # see https://github.com/easybuilders/easybuild-framework/issues/3376 ml_show_java18_stderr = '\n'.join([ "module-version Java/1.8.0_181 1.8", - "WARNING: Duplicate version symbol '1.8' found", + "WARNING: Symbolic version 'Java/1.8' already defined", "module-version Java/1.8.0_181 1.8", "-------------------------------------------------------------------", "/modulefiles/lang/Java/1.8.0_181:", From 94d0fa686c1b507ea2f6a50d54ecdccd6751880b Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sat, 31 Aug 2024 17:38:04 +0200 Subject: [PATCH 428/430] No error when using of non-existent modulepath dir on Modules 5+ Clarify comment in setup_categorized_hmns_modules test about the need for a modulepath directory to exist prior running "module use" on it. This is not required anymore starting Environment Modules 5.0. --- test/framework/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index e712e678a7..c20de50624 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -399,7 +399,7 @@ def setup_categorized_hmns_modules(self): src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules', 'CategorizedHMNS', mod_subdir) copy_dir(src_mod_path, os.path.join(mod_prefix, mod_subdir)) - # create empty module file directory to make C/Tcl modules happy + # create empty module file directory to make Environment Modules <5.0 happy mpi_pref = os.path.join(mod_prefix, 'MPI', 'GCC', '6.4.0-2.28', 'OpenMPI', '2.1.2') mkdir(os.path.join(mpi_pref, 'base')) From dc066099bbf21bd9ef443301c768b4d3ae5943b2 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sat, 31 Aug 2024 18:14:16 +0200 Subject: [PATCH 429/430] Update tests to use EnvironmentModules rather EnvironmentModulesC Update the test_avail_lists, test_show_config_cfg_levels and test_modules_tool_vs_syntax_check tests to make use of EnvironmentModules module tool rather deprecated EnvironmentModulesC. --- test/framework/options.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 6e7823f907..769e83b6f1 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -864,7 +864,7 @@ def test_avail_lists(self): os.close(fd) name_items = { - 'modules-tools': ['EnvironmentModulesC', 'Lmod'], + 'modules-tools': ['EnvironmentModules', 'Lmod'], 'module-naming-schemes': ['EasyBuildMNS', 'HierarchicalMNS', 'CategorizedHMNS'], } for (name, items) in name_items.items(): @@ -880,7 +880,7 @@ def test_avail_lists(self): info_msg = r"INFO List of supported %s:" % words self.assertTrue(re.search(info_msg, logtxt), "Info message with list of available %s" % words) for item in items: - res = re.findall(r"^\s*%s" % item, logtxt, re.M) + res = re.findall(r"^\s*%s\n" % item, logtxt, re.M) self.assertTrue(res, "%s is included in list of available %s" % (item, words)) # every item should only be mentioned once n = len(res) @@ -5315,19 +5315,19 @@ def test_show_config_cfg_levels(self): # configuring --modules-tool and --module-syntax on different levels should NOT cause problems # cfr. bug report https://github.com/easybuilders/easybuild-framework/issues/2564 - os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC' + os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModules' args = [ '--module-syntax=Tcl', '--show-config', ] # set init_config to False to avoid that eb_main (called by _run_mock_eb) re-initialises configuration - # this fails because $EASYBUILD_MODULES_TOOL=EnvironmentModulesC conflicts with default module syntax (Lua) + # this fails because $EASYBUILD_MODULES_TOOL=EnvironmentModules conflicts with default module syntax (Lua) stdout, _ = self._run_mock_eb(args, raise_error=True, redo_init_config=False) patterns = [ r"^# Current EasyBuild configuration", r"^module-syntax\s*\(C\) = Tcl", - r"^modules-tool\s*\(E\) = EnvironmentModulesC", + r"^modules-tool\s*\(E\) = EnvironmentModules", ] for pattern in patterns: regex = re.compile(pattern, re.M) @@ -5339,8 +5339,8 @@ def test_modules_tool_vs_syntax_check(self): # make sure default module syntax is used os.environ.pop('EASYBUILD_MODULE_SYNTAX', None) - # using EnvironmentModulesC modules tool with default module syntax (Lua) is a problem - os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC' + # using EnvironmentModules modules tool with default module syntax (Lua) is a problem + os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModules' args = ['--show-full-config'] error_pattern = "Generating Lua module files requires Lmod as modules tool" self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True) @@ -5348,10 +5348,10 @@ def test_modules_tool_vs_syntax_check(self): patterns = [ r"^# Current EasyBuild configuration", r"^module-syntax\s*\(C\) = Tcl", - r"^modules-tool\s*\(E\) = EnvironmentModulesC", + r"^modules-tool\s*\(E\) = EnvironmentModules", ] - # EnvironmentModulesC modules tool + Tcl module syntax is fine + # EnvironmentModules modules tool + Tcl module syntax is fine args.append('--module-syntax=Tcl') stdout, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, redo_init_config=False) for pattern in patterns: From 68191bddd05461702650f8ea8a2cad27bd922297 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 1 Sep 2024 07:23:45 +0200 Subject: [PATCH 430/430] Make EnvironmentModules inherits from ModulesTool As EnvironmentModulesTcl and EnvironmentModulesC classes are now deprecated, make EnvironmentModules directly inherits from ModulesTool. As EnvironmentModules class is now independent of the 2 deprecated classes, it will be easy to remove them from the code base. Specific methods run_module and update from EnvironmentModulesC and run_module, available and set_path_env_var from EnvironmentModulesTcl do not need to be ported to EnvironmentModules. The issues these methods handle do not affect Environment Modules v4+. Specific properties TERSE_OPTION, COMMAND_SHELL and VERSION_OPTION from EnvironmentModulesTcl do not need to be ported to EnvironmentModules. modulecmd.tcl is an executable script on Environment Modules v4+, so tclsh COMMAND_SHELL is not required to launch it (script's shebang triggers tclsh execution). Specific methods get_setenv_value_from_modulefile from EnvironmentModulesC and remove_module_path from EnvironmentModulesTcl have been ported to EnvironmentModules. --- easybuild/tools/modules.py | 31 ++++++++++++++++++++++++++----- test/framework/modules.py | 10 ++++------ test/framework/modulestool.py | 5 +++-- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 086a56f669..e986459e3f 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1317,14 +1317,14 @@ def remove_module_path(self, path, set_mod_paths=True): self.set_mod_paths() -class EnvironmentModules(EnvironmentModulesTcl): +class EnvironmentModules(ModulesTool): """Interface to Environment Modules 4.0+""" NAME = "Environment Modules" COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' REQ_VERSION_TCL_GETENV = '4.2.0' - DEPR_VERSION = '4.0.0' # needs to be set as EnvironmentModules inherits from EnvironmentModulesTcl + DEPR_VERSION = '4.0.0' MAX_VERSION = None REQ_VERSION_TCL_CHECK_GROUP = '4.6.0' REQ_VERSION_SAFE_AUTO_LOAD = '4.2.4' @@ -1420,14 +1420,35 @@ def get_setenv_value_from_modulefile(self, mod_name, var_name): # - line starts with 'setenv' # - whitespace (spaces & tabs) around variable name # - curly braces around value if it contain spaces - value = super(EnvironmentModules, self).get_setenv_value_from_modulefile(mod_name=mod_name, - var_name=var_name) + regex = re.compile(r'^setenv\s+%s\s+(?P.+)' % var_name, re.M) + value = self.get_value_from_modulefile(mod_name, regex, strict=False) if value: - value = value.strip('{}') + value = value.strip(' {}') return value + def remove_module_path(self, path, set_mod_paths=True): + """ + Remove specified module path (using 'module unuse'). + + :param path: path to remove from $MODULEPATH via 'unuse' + :param set_mod_paths: (re)set self.mod_paths + """ + # remove module path via 'module use' and make sure self.mod_paths is synced + # Environment Modules <5.0 keeps track of how often a path was added via 'module use', + # so we need to check to make sure it's really removed + path = normalize_path(path) + while True: + try: + # Unuse the path that is actually present in the environment + module_path = next(p for p in curr_module_paths() if normalize_path(p) == path) + except StopIteration: + break + self.unuse(module_path) + if set_mod_paths: + self.set_mod_paths() + def update(self): """Update after new modules were added.""" diff --git a/test/framework/modules.py b/test/framework/modules.py index 5a2026258f..5cd783694d 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -122,10 +122,10 @@ def test_run_module(self): error_pattern = "Module command '.*thisdoesnotmakesense' failed with exit code [1-9]" self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.run_module, 'thisdoesnotmakesense') - # we need to use a different error pattern here with EnvironmentModulesC, + # we need to use a different error pattern here with Environment Modules, # because a load of a non-existing module doesnt' trigger a non-zero exit code... # it will still fail though, just differently - if isinstance(self.modtool, EnvironmentModulesC): + if isinstance(self.modtool, EnvironmentModulesC) or isinstance(self.modtool, EnvironmentModules): error_pattern = "Unable to locate a modulefile for 'nosuchmodule/1.2.3'" else: error_pattern = "Module command '.*load nosuchmodule/1.2.3' failed with exit code [1-9]" @@ -213,10 +213,8 @@ def test_avail(self): # test modules include 3 GCC modules and one GCCcore module ms = self.modtool.available('GCC') expected = ['GCC/12.3.0', 'GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30'] - # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C do prefix matching - # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to Tcl/C impl., - # so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules - if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules): + # ancient Tcl-only Environment Modules tool does an exact match on module name, others do prefix matching + if not isinstance(self.modtool, EnvironmentModulesTcl): expected.extend(['GCCcore/12.3.0', 'GCCcore/6.2.0']) self.assertEqual(ms, expected) diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index c9396cf5cf..3caa1772ce 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -236,8 +236,9 @@ def test_environment_modules_specific(self): # pass (fake) full path to 'modulecmd.tcl' via $MODULES_CMD fake_path = os.path.join(self.test_installpath, 'libexec', 'modulecmd.tcl') fake_modulecmd_txt = '\n'.join([ - 'puts stderr {Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)}', - "puts {os.environ['FOO'] = 'foo'}", + '#!/bin/bash', + 'echo "Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)" >&2', + 'echo "os.environ[\'FOO\'] = \'foo\'"', ]) write_file(fake_path, fake_modulecmd_txt) os.chmod(fake_path, stat.S_IRUSR | stat.S_IXUSR)