Merge pull request #32 from smoors/fix_mod_install

fix module symlinks

bin/submit_build.py

@@ -30,7 +30,8 @@
 from build_tools.clusters import ARCHS, PARTITIONS
 from build_tools.filetools import APPS_BRUSSEL
 from build_tools import hooks_hydra
-from build_tools.hooks_hydra import SUBDIR_MODULES_BWRAP
+from build_tools.hooks_hydra import (SUBDIR_MODULES_BWRAP, SUFFIX_MODULES_PATH, SUFFIX_MODULES_SYMLINK,
+                                     VALID_MODULES_SUBDIRS)
 from build_tools.lmodtools import submit_lmod_cache_job
 from build_tools.softinstall import mk_job_name, submit_build_job
 
@@ -66,7 +67,6 @@ def main():
         'extra_mod_footer': None,
         'langcode': 'en_US.utf8',
         'lmod_cache': '1',
-        'subdir_modules_bwrap': SUBDIR_MODULES_BWRAP,
         'target_arch': None,
         'tmp': '/dev/shm',
     }
@@ -239,10 +239,21 @@ def main():
         ebconf['buildpath'] = os.path.join(job['tmp'], 'eb-submit-build')
 
         # common EB command line options
-        eb_options = ['--logtostdout', '--debug', '--module-extensions', '--zip-logs=bzip2', '--module-depends-on']
+        eb_options = [
+            '--logtostdout',
+            '--debug',
+            '--module-extensions',
+            '--zip-logs=bzip2',
+            '--module-depends-on',
+            f'--suffix-modules-path={SUFFIX_MODULES_PATH}',
+            f'--moduleclasses={",".join(os.path.join(x, SUFFIX_MODULES_SYMLINK) for x in VALID_MODULES_SUBDIRS)}',
+        ]
 
         if bwrap:
-            eb_options.extend([' --rebuild', f'--subdir-modules={job_options["subdir_modules_bwrap"]}'])
+            eb_options.extend([
+                '--rebuild',
+                f'--subdir-modules={SUBDIR_MODULES_BWRAP}',
+            ])
         else:
             # robot is not supported with bwrap
             eb_options.append('--robot')

src/build_tools/hooks_hydra.py

@@ -18,7 +18,6 @@
 """
 
 import os
-from pathlib import Path
 import sys
 import time
 
@@ -28,7 +27,7 @@
 from easybuild.framework.easyconfig.easyconfig import letter_dir_for, get_toolchain_hierarchy
 from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import BuildOptions, ConfigurationVariables, source_paths, update_build_option
+from easybuild.tools.config import build_option, ConfigurationVariables, source_paths, update_build_option
 from easybuild.tools.filetools import mkdir
 from easybuild.tools.hooks import SANITYCHECK_STEP
 
@@ -68,10 +67,17 @@
 VALID_TCS = ['foss', 'intel', 'gomkl', 'gimkl', 'gimpi']
 
 SUBDIR_MODULES_BWRAP = '.modules_bwrap'
+SUFFIX_MODULES_PATH = 'collection'
+SUFFIX_MODULES_SYMLINK = 'all'
 
 
 def get_tc_versions():
     " build dict of valid (sub)toolchain-version combinations per valid generation "
+
+    # temporarily disable hooks to avoid infinite recursion when calling get_toolchain_hierarchy()
+    hooks = build_option('hooks')
+    update_build_option('hooks', None)
+
     tc_versions = {}
     for toolcgen in VALID_TCGENS:
         tc_versions[toolcgen] = []
@@ -82,6 +88,7 @@ def get_tc_versions():
                 # skip if no easyconfig found for toolchain-version
                 pass
 
+    update_build_option('hooks', hooks)
     return tc_versions
 
 
@@ -118,63 +125,19 @@ def calc_tc_gen(name, version, tcname, tcversion, easyblock):
     return False, log_msg
 
 
-def update_module_install_paths(self):
-    """
-    update module install paths unless subdir-modules uption is specified "
-    default subdir_modules config var = 'modules'
-    here we set it to 'modules/<subdir>', where subdir can be any of VALID_MODULES_SUBDIRS
-    exception: with bwrap it is set to SUBDIR_MODULES_BWRAP
-    """
-    configvars = ConfigurationVariables()
-    subdir_modules = list(Path(configvars['subdir_modules']).parts)
-
-    do_bwrap = subdir_modules == [SUBDIR_MODULES_BWRAP]
-
-    log_format_msg = '[pre-fetch hook] Format of option subdir-modules %s is not valid. Must be modules/<subdir>'
-    if len(subdir_modules) not in [1, 2]:
-        raise EasyBuildError(log_format_msg, os.path.join(*subdir_modules))
-
-    if not (subdir_modules[0] == 'modules' or subdir_modules != ['modules'] or do_bwrap):
-        raise EasyBuildError(log_format_msg, os.path.join(*subdir_modules))
-
-    if len(subdir_modules) == 2:
-        subdir = subdir_modules[1]
+def update_moduleclass(ec):
+    "update the moduleclass of an easyconfig to <tc_gen>/all"
+    tc_gen, log_msg = calc_tc_gen(
+        ec.name, ec.version, ec.toolchain.name, ec.toolchain.version, ec.easyblock)
 
-        if subdir not in VALID_MODULES_SUBDIRS:
-            log_msg = "[pre-fetch hook] Specified modules subdir %s is not valid. Choose one of %s"
-            raise EasyBuildError(log_msg, subdir, VALID_MODULES_SUBDIRS)
+    if not tc_gen:
+        raise EasyBuildError("[parse hook] " + log_msg)
 
-        log_msg = "[pre-fetch hook] Option subdir-modules was set to %s, not updating module install paths"
-        self.log.info(log_msg, subdir_modules)
-        return
+    ec.log.info("[parse hook] " + log_msg)
 
-    subdir, log_msg = calc_tc_gen(
-        self.name, self.version, self.toolchain.name, self.toolchain.version, self.cfg.easyblock)
+    ec['moduleclass'] = os.path.join(tc_gen, SUFFIX_MODULES_SYMLINK)
 
-    if not subdir:
-        raise EasyBuildError("[pre-fetch hook] " + log_msg)
-
-    self.log.info("[pre-fetch hook] " + log_msg)
-
-    mod_filepath = Path(self.mod_filepath).parts
-
-    if do_bwrap:
-        self.log.info("[pre-fetch hook] Installing in new namespace with bwrap")
-        real_mod_filepath = Path().joinpath(*mod_filepath[:-4], 'modules', subdir, *mod_filepath[-3:]).as_posix()
-
-        # write the real module file path to stderr
-        # after installation, the module file is copied to the real path
-        sys.stderr.write(f'BUILD_TOOLS: real_mod_filepath {real_mod_filepath}\n')
-        return
-
-    # insert subdir into self.installdir_mod and self.mod_filepath
-    installdir_mod = Path(self.installdir_mod).parts
-    self.installdir_mod = Path().joinpath(*installdir_mod[:-1], subdir, installdir_mod[-1]).as_posix()
-    self.log.info('[pre-fetch hook] Updated installdir_mod to %s', self.installdir_mod)
-
-    real_mod_filepath = Path().joinpath(*mod_filepath[:-3], subdir, *mod_filepath[-3:]).as_posix()
-    self.mod_filepath = real_mod_filepath
-    self.log.info('[pre-fetch hook] Updated mod_filepath to %s', self.mod_filepath)
+    ec.log.info("[parse hook] updated moduleclass for %s to %s", os.path.basename(ec.path), ec['moduleclass'])
 
 
 def acquire_fetch_lock(self):
@@ -225,10 +188,13 @@ def release_fetch_lock(self):
 def parse_hook(ec, *args, **kwargs):  # pylint: disable=unused-argument
     """Alter build options and easyconfig parameters"""
 
+    if not ec['moduleclass'].endswith(f'/{SUFFIX_MODULES_SYMLINK}'):
+        update_moduleclass(ec)
+
     # disable robot for bwrap
     # must be done in a hook in case robot is set in an easystack, which takes precedence over cmd line options
     subdir_modules = ConfigurationVariables()['subdir_modules']
-    robot = BuildOptions()['robot']
+    robot = build_option('robot')
     if subdir_modules == SUBDIR_MODULES_BWRAP and robot is not None:
         update_build_option('robot', None)
         ec.log.warning("[parse hook] Disabled robot for bwrap")
@@ -315,7 +281,6 @@ def parse_hook(ec, *args, **kwargs):  # pylint: disable=unused-argument
 
 def pre_fetch_hook(self):
     """Hook at pre-fetch level"""
-    update_module_install_paths(self)
     acquire_fetch_lock(self)
 
 

src/build_tools/jobtemplate.py

@@ -20,7 +20,8 @@
 
 from string import Template
 
-BUILD_JOB = """#!/bin/bash -l
+BUILD_JOB = """\
+#!/bin/bash -l
 #SBATCH --job-name=${job_name}
 #SBATCH --output="%x-%j.out"
 #SBATCH --error="%x-%j.err"
@@ -38,6 +39,12 @@
 export PATH=$$PREFIX_EB/easybuild-framework:$$PATH
 export PYTHONPATH=$$PREFIX_EB/easybuild-easyconfigs:$$PREFIX_EB/easybuild-easyblocks:$$PREFIX_EB/easybuild-framework:$$PREFIX_EB/vsc-base/lib
 
+SUBDIR_MODULES="modules"
+SUBDIR_MODULES_BWRAP=".modules_bwrap"
+SUFFIX_MODULES_PATH="collection"
+SUFFIX_MODULES_SYMLINK="all"
+
+
 # make build directory
 if [ -z $$SLURM_JOB_ID ]; then
     export TMPDIR=${tmp}/$$USER/
@@ -56,7 +63,7 @@
 if [ "${bwrap}" == 1 ]; then
     echo "BUILD_TOOLS: installing with bwrap"
     output=$$(EASYBUILD_ROBOT_PATHS=${robot_paths} EASYBUILD_IGNORE_INDEX=1 ec2ml.py ${easyconfig}) || { echo "ERROR: ec2ml.py failed"; exit 1; }
-    echo "BUILD_TOOLS: get_module_from_easyconfig.py output: $$output"
+    echo "BUILD_TOOLS: ec2ml.py ${easyconfig} output: $$output"
     while read -r key value; do
         [ "$$key" == "full_mod_name" ] && { modname=$${value%/*}; modversion=$${value#*/}; break; }
     done <<< "$$output"
@@ -65,9 +72,7 @@
     appsmnt="/vscmnt/brussel_pixiu_apps/_apps_brussel"
     softbwrap="/apps/brussel/bwrap/$$VSC_OS_LOCAL/${target_arch}/software/$$modname"
     softreal="$$appsmnt/$$VSC_OS_LOCAL/${target_arch}/software/$$modname"
-    modbwrap="/apps/brussel/$$VSC_OS_LOCAL/${target_arch}/${subdir_modules_bwrap}/all/$$modname"
     mkdir -p "$$softbwrap"
-    mkdir -p "$$modbwrap"
     bwrap_cmd=(
         bwrap
         --bind / /
@@ -94,18 +99,28 @@
 fi
 
 if [ "${bwrap}" == 1 ]; then
-    dest_modfile=$$(grep "^BUILD_TOOLS: real_mod_filepath" "$$eb_stderr" | cut -d " " -f 3) || { echo "ERROR: failed to obtain destination module file path"; exit 1; }
     source_installdir="$$softbwrap/$$modversion/"
     dest_installdir="$$softreal/$$modversion/"
-    source_modfile="$$modbwrap/$$modversion.lua"
-    echo "BUILD_TOOLS: source/destination install dir: $$source_installdir $$dest_installdir"
-    echo "BUILD_TOOLS: source/destination module file: $$source_modfile $$dest_modfile"
+    installbase="/apps/brussel/$$VSC_OS_LOCAL/${target_arch}"
+    source_modfile="$$installbase/$$SUBDIR_MODULES_BWRAP/$$SUFFIX_MODULES_PATH/$$modname/$$modversion.lua"
+    source_modsymlink=$$(echo $$installbase/$$SUBDIR_MODULES_BWRAP/*/$$SUFFIX_MODULES_SYMLINK/$$modname/$$modversion.lua)
+    dest_modfile="$$installbase/$$SUBDIR_MODULES/$$SUFFIX_MODULES_PATH/$$modname/$$modversion.lua"
+    dest_modsymlink=$${source_modsymlink/$$installbase\/$$SUBDIR_MODULES_BWRAP\//$$installbase\/$$SUBDIR_MODULES\/}
+    echo "BUILD_TOOLS: source/dest install dir: $$source_installdir $$dest_installdir"
+    echo "BUILD_TOOLS: source/dest module file: $$source_modfile $$dest_modfile"
+    echo "BUILD_TOOLS: source/dest module symlink: $$source_modsymlink $$dest_modsymlink"
     test -d "$$source_installdir" || { echo "ERROR: source install dir does not exist"; exit 1; }
     test -n "$$(ls -A $$source_installdir)" || { echo "ERROR: source install dir is empty"; exit 1; }
     test -s "$$source_modfile" || { echo "ERROR: source module file does not exist or is empty"; exit 1; }
+    test $$(readlink "$$source_modsymlink") == "$$source_modfile" || { echo "ERROR: source module symlink does not link to correct file"; exit 1; }
+    mkdir -p $$(dirname "$$dest_modfile") $$(dirname "$$dest_modsymlink")
+    tempfile=$$(mktemp -p /tmp)
     rsync -a --link-dest="$$source_installdir" "$$source_installdir" "$$dest_installdir" || { echo "ERROR: failed to copy install dir"; exit 1; }
-    rsync -a --link-dest="$$modbwrap" "$$source_modfile" "$$dest_modfile" || { echo "ERROR: failed to copy module file"; exit 1; }
-    rm -rf "$$source_installdir" "$$source_modfile"
+    cp -p "$$source_modfile" "$$dest_modfile"
+    ln -sf "$$dest_modfile" "$$tempfile"
+    mv -f "$$tempfile" "$$dest_modsymlink"
+    test $$(readlink "$$dest_modsymlink") == "$$dest_modfile" || { echo "ERROR: failed to create symlink to module file"; exit 1; }
+    rm -rf "$$source_installdir" "$$source_modfile" "$$source_modsymlink"
     echo "BUILD_TOOLS: installation moved from bwrap to real location"
 fi
 

src/build_tools/package.py

@@ -16,7 +16,7 @@
 @author: Alex Domingo (Vrije Universiteit Brussel)
 """
 
-VERSION = '4.0.0'
+VERSION = '4.0.1'
 
 AUTHOR = {
     'wp': 'Ward Poelmans',

tests/conftest.py

@@ -31,9 +31,9 @@ def pytest_addoption(parser):
 def get_module_cmd(request):
     fromsource = request.config.getoption('fromsource')
     if fromsource:
-        return '../bin/get_module_from_easyconfig.py'
+        return '../bin/ec2ml.py'
     else:
-        return 'get_module_from_easyconfig.py'
+        return 'ec2ml.py'
 
 
 @pytest.fixture

tests/input/build_job_01.sh

@@ -16,6 +16,12 @@ export LANG=C
 export PATH=$PREFIX_EB/easybuild-framework:$PATH
 export PYTHONPATH=$PREFIX_EB/easybuild-easyconfigs:$PREFIX_EB/easybuild-easyblocks:$PREFIX_EB/easybuild-framework:$PREFIX_EB/vsc-base/lib
 
+SUBDIR_MODULES="modules"
+SUBDIR_MODULES_BWRAP=".modules_bwrap"
+SUFFIX_MODULES_PATH="collection"
+SUFFIX_MODULES_SYMLINK="all"
+
+
 # make build directory
 if [ -z $SLURM_JOB_ID ]; then
     export TMPDIR=/tmp/eb-test-build/$USER/
@@ -34,7 +40,7 @@ EB='eb'
 if [ "0" == 1 ]; then
     echo "BUILD_TOOLS: installing with bwrap"
     output=$(EASYBUILD_ROBOT_PATHS=/some/path EASYBUILD_IGNORE_INDEX=1 ec2ml.py zlib-1.2.11.eb) || { echo "ERROR: ec2ml.py failed"; exit 1; }
-    echo "BUILD_TOOLS: get_module_from_easyconfig.py output: $output"
+    echo "BUILD_TOOLS: ec2ml.py zlib-1.2.11.eb output: $output"
     while read -r key value; do
         [ "$key" == "full_mod_name" ] && { modname=${value%/*}; modversion=${value#*/}; break; }
     done <<< "$output"
@@ -43,9 +49,7 @@ if [ "0" == 1 ]; then
     appsmnt="/vscmnt/brussel_pixiu_apps/_apps_brussel"
     softbwrap="/apps/brussel/bwrap/$VSC_OS_LOCAL/skylake/software/$modname"
     softreal="$appsmnt/$VSC_OS_LOCAL/skylake/software/$modname"
-    modbwrap="/apps/brussel/$VSC_OS_LOCAL/skylake/.modules_bwrap/all/$modname"
     mkdir -p "$softbwrap"
-    mkdir -p "$modbwrap"
     bwrap_cmd=(
         bwrap
         --bind / /
@@ -72,18 +76,28 @@ if [ $ec -ne 0 ]; then
 fi
 
 if [ "0" == 1 ]; then
-    dest_modfile=$(grep "^BUILD_TOOLS: real_mod_filepath" "$eb_stderr" | cut -d " " -f 3) || { echo "ERROR: failed to obtain destination module file path"; exit 1; }
     source_installdir="$softbwrap/$modversion/"
     dest_installdir="$softreal/$modversion/"
-    source_modfile="$modbwrap/$modversion.lua"
-    echo "BUILD_TOOLS: source/destination install dir: $source_installdir $dest_installdir"
-    echo "BUILD_TOOLS: source/destination module file: $source_modfile $dest_modfile"
+    installbase="/apps/brussel/$VSC_OS_LOCAL/skylake"
+    source_modfile="$installbase/$SUBDIR_MODULES_BWRAP/$SUFFIX_MODULES_PATH/$modname/$modversion.lua"
+    source_modsymlink=$(echo $installbase/$SUBDIR_MODULES_BWRAP/*/$SUFFIX_MODULES_SYMLINK/$modname/$modversion.lua)
+    dest_modfile="$installbase/$SUBDIR_MODULES/$SUFFIX_MODULES_PATH/$modname/$modversion.lua"
+    dest_modsymlink=${source_modsymlink/$installbase\/$SUBDIR_MODULES_BWRAP\//$installbase\/$SUBDIR_MODULES\/}
+    echo "BUILD_TOOLS: source/dest install dir: $source_installdir $dest_installdir"
+    echo "BUILD_TOOLS: source/dest module file: $source_modfile $dest_modfile"
+    echo "BUILD_TOOLS: source/dest module symlink: $source_modsymlink $dest_modsymlink"
     test -d "$source_installdir" || { echo "ERROR: source install dir does not exist"; exit 1; }
     test -n "$(ls -A $source_installdir)" || { echo "ERROR: source install dir is empty"; exit 1; }
     test -s "$source_modfile" || { echo "ERROR: source module file does not exist or is empty"; exit 1; }
+    test $(readlink "$source_modsymlink") == "$source_modfile" || { echo "ERROR: source module symlink does not link to correct file"; exit 1; }
+    mkdir -p $(dirname "$dest_modfile") $(dirname "$dest_modsymlink")
+    tempfile=$(mktemp -p /tmp)
     rsync -a --link-dest="$source_installdir" "$source_installdir" "$dest_installdir" || { echo "ERROR: failed to copy install dir"; exit 1; }
-    rsync -a --link-dest="$modbwrap" "$source_modfile" "$dest_modfile" || { echo "ERROR: failed to copy module file"; exit 1; }
-    rm -rf "$source_installdir" "$source_modfile"
+    cp -p "$source_modfile" "$dest_modfile"
+    ln -sf "$dest_modfile" "$tempfile"
+    mv -f "$tempfile" "$dest_modsymlink"
+    test $(readlink "$dest_modsymlink") == "$dest_modfile" || { echo "ERROR: failed to create symlink to module file"; exit 1; }
+    rm -rf "$source_installdir" "$source_modfile" "$source_modsymlink"
     echo "BUILD_TOOLS: installation moved from bwrap to real location"
 fi