diff --git a/bin/submit_build.py b/bin/submit_build.py index b8f207b..903a3c5 100755 --- a/bin/submit_build.py +++ b/bin/submit_build.py @@ -24,20 +24,27 @@ import sys from vsc.utils import fancylogger -from vsc.utils.script_tools import SimpleOption from vsc.utils.run import RunNoShell +from vsc.utils.script_tools import SimpleOption +from build_tools import hooks_hydra from build_tools.clusters import ARCHS, PARTITIONS from build_tools.filetools import APPS_BRUSSEL -from build_tools import hooks_hydra -from build_tools.hooks_hydra import (SUBDIR_MODULES_BWRAP, SUFFIX_MODULES_PATH, SUFFIX_MODULES_SYMLINK, - VALID_MODULES_SUBDIRS) +from build_tools.hooks_hydra import ( + SUBDIR_MODULES_BWRAP, + SUFFIX_MODULES_PATH, + SUFFIX_MODULES_SYMLINK, + VALID_MODULES_SUBDIRS, +) from build_tools.lmodtools import submit_lmod_cache_job from build_tools.softinstall import mk_job_name, submit_build_job +logger = fancylogger.getLogger() +fancylogger.logToScreen(True) +fancylogger.setLogLevelInfo() # repositories with easyconfigs -VSCSOFTSTACK_ROOT = os.path.expanduser("~/vsc-software-stack") +VSCSOFTSTACK_ROOT = os.path.join(os.path.dirname(os.getenv("VIRTUAL_ENV", "")), "vsc-software-stack") EASYCONFIG_REPOS = [ # our site repo (https://github.com/vscentrum/vsc-software-stack/tree/site-vub) os.path.join("site-vub", "easyconfigs"), @@ -46,10 +53,6 @@ ] EASYBLOCK_REPO = os.path.join("site-vub", "easyblocks", "*", "*.py") -logger = fancylogger.getLogger() -fancylogger.logToScreen(True) -fancylogger.setLogLevelInfo() - DEFAULT_ARCHS = [arch for (arch, prop) in ARCHS.items() if prop['default']] LOCAL_ARCH = os.getenv('VSC_ARCH_LOCAL', '') + os.getenv('VSC_ARCH_SUFFIX', '') if LOCAL_ARCH not in ARCHS: @@ -120,6 +123,13 @@ def main(): logger.error("No easyconfig is given...") sys.exit(1) + if not os.path.isdir(VSCSOFTSTACK_ROOT): + logger.error( + f"Cannot locate 'vsc-software-stack' repo in: {VSCSOFTSTACK_ROOT} - " + "Please clone that repo in the parent folder of your virtual environment directory" + ) + sys.exit(1) + easyconfig = ' '.join(opts.args) job['easyconfig'] = easyconfig logger.info("Preparing to install %s", easyconfig) diff --git a/src/build_tools/jobtemplate.py b/src/build_tools/jobtemplate.py index cb62bb3..9bae82d 100644 --- a/src/build_tools/jobtemplate.py +++ b/src/build_tools/jobtemplate.py @@ -31,20 +31,18 @@ #SBATCH --gpus-per-node=${gpus} #SBATCH --partition=${partition} -test -n "$$PREFIX_EB" || { echo "ERROR: environment variable PREFIX_EB not set"; exit 1; } +# activate build_tools virtual environment +source "$$VSC_SCRATCH_VO_USER/EB4/eb4env/bin/activate" # set environment export BUILD_TOOLS_LOAD_DUMMY_MODULES=1 export LANG=${langcode} -export PATH=$$PREFIX_EB/easybuild-framework:$$PATH -export PYTHONPATH=$$PREFIX_EB/easybuild-easyconfigs:$$PREFIX_EB/easybuild-easyblocks:$$PREFIX_EB/easybuild-framework:$$PREFIX_EB/vsc-base/lib SUBDIR_MODULES="modules" SUBDIR_MODULES_BWRAP=".modules_bwrap" SUFFIX_MODULES_PATH="collection" SUFFIX_MODULES_SYMLINK="all" - # make build directory if [ -z $$SLURM_JOB_ID ]; then export TMPDIR=${tmp}/$$USER/ diff --git a/src/build_tools/package.py b/src/build_tools/package.py index 0c5b485..54edb40 100644 --- a/src/build_tools/package.py +++ b/src/build_tools/package.py @@ -16,7 +16,7 @@ @author: Alex Domingo (Vrije Universiteit Brussel) """ -VERSION = '4.0.2' +VERSION = '4.0.3' AUTHOR = { 'wp': 'Ward Poelmans', diff --git a/tests/input/build_job_01.sh b/tests/input/build_job_01.sh index 5c8f304..65ac816 100644 --- a/tests/input/build_job_01.sh +++ b/tests/input/build_job_01.sh @@ -8,20 +8,18 @@ #SBATCH --gpus-per-node=0 #SBATCH --partition=skylake_mpi -test -n "$PREFIX_EB" || { echo "ERROR: environment variable PREFIX_EB not set"; exit 1; } +# activate build_tools virtual environment +source "$VSC_SCRATCH_VO_USER/EB4/eb4env/bin/activate" # set environment export BUILD_TOOLS_LOAD_DUMMY_MODULES=1 export LANG=C -export PATH=$PREFIX_EB/easybuild-framework:$PATH -export PYTHONPATH=$PREFIX_EB/easybuild-easyconfigs:$PREFIX_EB/easybuild-easyblocks:$PREFIX_EB/easybuild-framework:$PREFIX_EB/vsc-base/lib SUBDIR_MODULES="modules" SUBDIR_MODULES_BWRAP=".modules_bwrap" SUFFIX_MODULES_PATH="collection" SUFFIX_MODULES_SYMLINK="all" - # make build directory if [ -z $SLURM_JOB_ID ]; then export TMPDIR=/tmp/eb-test-build/$USER/ diff --git a/tests/input/build_job_02.sh b/tests/input/build_job_02.sh index 1641b49..08c0b2c 100644 --- a/tests/input/build_job_02.sh +++ b/tests/input/build_job_02.sh @@ -8,20 +8,18 @@ #SBATCH --gpus-per-node=1 #SBATCH --partition=ampere_gpu -test -n "$PREFIX_EB" || { echo "ERROR: environment variable PREFIX_EB not set"; exit 1; } +# activate build_tools virtual environment +source "$VSC_SCRATCH_VO_USER/EB4/eb4env/bin/activate" # set environment export BUILD_TOOLS_LOAD_DUMMY_MODULES=1 export LANG=C -export PATH=$PREFIX_EB/easybuild-framework:$PATH -export PYTHONPATH=$PREFIX_EB/easybuild-easyconfigs:$PREFIX_EB/easybuild-easyblocks:$PREFIX_EB/easybuild-framework:$PREFIX_EB/vsc-base/lib SUBDIR_MODULES="modules" SUBDIR_MODULES_BWRAP=".modules_bwrap" SUFFIX_MODULES_PATH="collection" SUFFIX_MODULES_SYMLINK="all" - # make build directory if [ -z $SLURM_JOB_ID ]; then export TMPDIR=/tmp/eb-test-build/$USER/