switch to virtual environment with EasyBuild and build_tools

bin/submit_build.py

@@ -24,20 +24,27 @@
 import sys
 
 from vsc.utils import fancylogger
-from vsc.utils.script_tools import SimpleOption
 from vsc.utils.run import RunNoShell
+from vsc.utils.script_tools import SimpleOption
 
+from build_tools import hooks_hydra
 from build_tools.clusters import ARCHS, PARTITIONS
 from build_tools.filetools import APPS_BRUSSEL
-from build_tools import hooks_hydra
-from build_tools.hooks_hydra import (SUBDIR_MODULES_BWRAP, SUFFIX_MODULES_PATH, SUFFIX_MODULES_SYMLINK,
-                                     VALID_MODULES_SUBDIRS)
+from build_tools.hooks_hydra import (
+    SUBDIR_MODULES_BWRAP,
+    SUFFIX_MODULES_PATH,
+    SUFFIX_MODULES_SYMLINK,
+    VALID_MODULES_SUBDIRS,
+)
 from build_tools.lmodtools import submit_lmod_cache_job
 from build_tools.softinstall import mk_job_name, submit_build_job
 
+logger = fancylogger.getLogger()
+fancylogger.logToScreen(True)
+fancylogger.setLogLevelInfo()
 
 # repositories with easyconfigs
-VSCSOFTSTACK_ROOT = os.path.expanduser("~/vsc-software-stack")
+VSCSOFTSTACK_ROOT = os.path.join(os.path.dirname(os.getenv("VIRTUAL_ENV", "")), "vsc-software-stack")
 EASYCONFIG_REPOS = [
     # our site repo (https://github.com/vscentrum/vsc-software-stack/tree/site-vub)
     os.path.join("site-vub", "easyconfigs"),
@@ -46,10 +53,6 @@
 ]
 EASYBLOCK_REPO = os.path.join("site-vub", "easyblocks", "*", "*.py")
 
-logger = fancylogger.getLogger()
-fancylogger.logToScreen(True)
-fancylogger.setLogLevelInfo()
-
 DEFAULT_ARCHS = [arch for (arch, prop) in ARCHS.items() if prop['default']]
 LOCAL_ARCH = os.getenv('VSC_ARCH_LOCAL', '') + os.getenv('VSC_ARCH_SUFFIX', '')
 if LOCAL_ARCH not in ARCHS:
@@ -120,6 +123,13 @@ def main():
         logger.error("No easyconfig is given...")
         sys.exit(1)
 
+    if not os.path.isdir(VSCSOFTSTACK_ROOT):
+        logger.error(
+            f"Cannot locate 'vsc-software-stack' repo in: {VSCSOFTSTACK_ROOT} - "
+            "Please clone that repo in the parent folder of your virtual environment directory"
+        )
+        sys.exit(1)
+
     easyconfig = ' '.join(opts.args)
     job['easyconfig'] = easyconfig
     logger.info("Preparing to install %s", easyconfig)

src/build_tools/jobtemplate.py

@@ -31,20 +31,18 @@
 #SBATCH --gpus-per-node=${gpus}
 #SBATCH --partition=${partition}
 
-test -n "$$PREFIX_EB" || { echo "ERROR: environment variable PREFIX_EB not set"; exit 1; }
+# activate build_tools virtual environment
+source "$$VSC_SCRATCH_VO_USER/EB4/eb4env/bin/activate"
 
 # set environment
 export BUILD_TOOLS_LOAD_DUMMY_MODULES=1
 export LANG=${langcode}
-export PATH=$$PREFIX_EB/easybuild-framework:$$PATH
-export PYTHONPATH=$$PREFIX_EB/easybuild-easyconfigs:$$PREFIX_EB/easybuild-easyblocks:$$PREFIX_EB/easybuild-framework:$$PREFIX_EB/vsc-base/lib
 
 SUBDIR_MODULES="modules"
 SUBDIR_MODULES_BWRAP=".modules_bwrap"
 SUFFIX_MODULES_PATH="collection"
 SUFFIX_MODULES_SYMLINK="all"
 
-
 # make build directory
 if [ -z $$SLURM_JOB_ID ]; then
     export TMPDIR=${tmp}/$$USER/

src/build_tools/package.py

@@ -16,7 +16,7 @@
 @author: Alex Domingo (Vrije Universiteit Brussel)
 """
 
-VERSION = '4.0.2'
+VERSION = '4.0.3'
 
 AUTHOR = {
     'wp': 'Ward Poelmans',

tests/input/build_job_01.sh

@@ -8,20 +8,18 @@
 #SBATCH --gpus-per-node=0
 #SBATCH --partition=skylake_mpi
 
-test -n "$PREFIX_EB" || { echo "ERROR: environment variable PREFIX_EB not set"; exit 1; }
+# activate build_tools virtual environment
+source "$VSC_SCRATCH_VO_USER/EB4/eb4env/bin/activate"
 
 # set environment
 export BUILD_TOOLS_LOAD_DUMMY_MODULES=1
 export LANG=C
-export PATH=$PREFIX_EB/easybuild-framework:$PATH
-export PYTHONPATH=$PREFIX_EB/easybuild-easyconfigs:$PREFIX_EB/easybuild-easyblocks:$PREFIX_EB/easybuild-framework:$PREFIX_EB/vsc-base/lib
 
 SUBDIR_MODULES="modules"
 SUBDIR_MODULES_BWRAP=".modules_bwrap"
 SUFFIX_MODULES_PATH="collection"
 SUFFIX_MODULES_SYMLINK="all"
 
-
 # make build directory
 if [ -z $SLURM_JOB_ID ]; then
     export TMPDIR=/tmp/eb-test-build/$USER/

tests/input/build_job_02.sh

@@ -8,20 +8,18 @@
 #SBATCH --gpus-per-node=1
 #SBATCH --partition=ampere_gpu
 
-test -n "$PREFIX_EB" || { echo "ERROR: environment variable PREFIX_EB not set"; exit 1; }
+# activate build_tools virtual environment
+source "$VSC_SCRATCH_VO_USER/EB4/eb4env/bin/activate"
 
 # set environment
 export BUILD_TOOLS_LOAD_DUMMY_MODULES=1
 export LANG=C
-export PATH=$PREFIX_EB/easybuild-framework:$PATH
-export PYTHONPATH=$PREFIX_EB/easybuild-easyconfigs:$PREFIX_EB/easybuild-easyblocks:$PREFIX_EB/easybuild-framework:$PREFIX_EB/vsc-base/lib
 
 SUBDIR_MODULES="modules"
 SUBDIR_MODULES_BWRAP=".modules_bwrap"
 SUFFIX_MODULES_PATH="collection"
 SUFFIX_MODULES_SYMLINK="all"
 
-
 # make build directory
 if [ -z $SLURM_JOB_ID ]; then
     export TMPDIR=/tmp/eb-test-build/$USER/