diff --git a/scabha/basetypes.py b/scabha/basetypes.py index f541dd4b..d949177c 100644 --- a/scabha/basetypes.py +++ b/scabha/basetypes.py @@ -142,6 +142,9 @@ def get_filelikes(dtype, value, filelikes=None): filelikes = set() if filelikes is None else filelikes + if value is UNSET or type(value) is UNSET: + return [] + origin = get_origin(dtype) args = get_args(dtype) diff --git a/scabha/configuratt/deps.py b/scabha/configuratt/deps.py index 81191cd1..b6d9fd37 100644 --- a/scabha/configuratt/deps.py +++ b/scabha/configuratt/deps.py @@ -18,6 +18,7 @@ class FailRecord(object): origin: Optional[str] = None modulename: Optional[str] = None fname: Optional[str] = None + warn: bool = True @dataclass class RequirementRecord(object): diff --git a/scabha/configuratt/resolvers.py b/scabha/configuratt/resolvers.py index 96e3d38a..62feee18 100644 --- a/scabha/configuratt/resolvers.py +++ b/scabha/configuratt/resolvers.py @@ -222,8 +222,11 @@ def load_include_files(keyword): if match: incl = match.group(1) flags = set([x.strip().lower() for x in match.group(2).split(",")]) + warn = 'warn' in flags + optional = 'optional' in flags else: flags = {} + warn = optional = False # check for (location)filename.yaml or (location)/filename.yaml style match = re.match("^\\((.+)\\)/?(.+)$", incl) @@ -232,9 +235,9 @@ def load_include_files(keyword): if modulename.startswith("."): filename = os.path.join(os.path.dirname(pathname), modulename, filename) if not os.path.exists(filename): - if 'optional' in flags: - dependencies.add_fail(FailRecord(filename, pathname)) - if 'warn' in flags: + if optional: + dependencies.add_fail(FailRecord(filename, pathname,warn=warn)) + if warn: print(f"Warning: unable to find optional include {incl} ({filename})") continue raise ConfigurattError(f"{errloc}: {keyword} {incl} does not exist") @@ -242,27 +245,42 @@ def load_include_files(keyword): try: mod = importlib.import_module(modulename) except ImportError as exc: - if 'optional' in flags: - dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, fname=filename)) - if 'warn' in flags: + if optional: + dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, + fname=filename, warn=warn)) + if warn: print(f"Warning: unable to import module for optional include {incl}") continue raise ConfigurattError(f"{errloc}: {keyword} {incl}: can't import {modulename} ({exc})") - - filename = os.path.join(os.path.dirname(mod.__file__), filename) + if mod.__file__ is not None: + path = os.path.dirname(mod.__file__) + else: + path = getattr(mod, '__path__', None) + if path is None: + if optional: + dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, + fname=filename, warn=warn)) + if warn: + print(f"Warning: unable to resolve path for optional include {incl}, does {modulename} contain __init__.py?") + continue + raise ConfigurattError(f"{errloc}: {keyword} {incl}: can't resolve path for {modulename}, does it contain __init__.py?") + path = path[0] + + filename = os.path.join(path, filename) if not os.path.exists(filename): - if 'optional' in flags: - dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, fname=filename)) - if 'warn' in flags: + if optional: + dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, + fname=filename, warn=warn)) + if warn: print(f"Warning: unable to find optional include {incl}") continue raise ConfigurattError(f"{errloc}: {keyword} {incl}: {filename} does not exist") # absolute path -- one candidate elif os.path.isabs(incl): if not os.path.exists(incl): - if 'optional' in flags: - dependencies.add_fail(FailRecord(incl, pathname)) - if 'warn' in flags: + if optional: + dependencies.add_fail(FailRecord(incl, pathname, warn=warn)) + if warn: print(f"Warning: unable to find optional include {incl}") continue raise ConfigurattError(f"{errloc}: {keyword} {incl} does not exist") @@ -275,9 +293,9 @@ def load_include_files(keyword): if os.path.exists(filename): break else: - if 'optional' in flags: - dependencies.add_fail(FailRecord(incl, pathname)) - if 'warn' in flags: + if optional: + dependencies.add_fail(FailRecord(incl, pathname, warn=warn)) + if warn: print(f"Warning: unable to find optional include {incl}") continue raise ConfigurattError(f"{errloc}: {keyword} {incl} not found in {':'.join(paths)}") diff --git a/stimela/backends/flavours/python_flavours.py b/stimela/backends/flavours/python_flavours.py index 34dc2122..88b4a485 100644 --- a/stimela/backends/flavours/python_flavours.py +++ b/stimela/backends/flavours/python_flavours.py @@ -51,13 +51,15 @@ def get_python_interpreter_args(cab: Cab, subst: Dict[str, Any], virtual_env: Op # get virtual env, if specified if virtual_env: virtual_env = os.path.expanduser(virtual_env) - interpreter = f"{virtual_env}/bin/python" + interpreter = f"{virtual_env}/bin/{cab.flavour.interpreter_binary}" if not os.path.isfile(interpreter): - raise CabValidationError(f"virtual environment {virtual_env} doesn't exist") + raise CabValidationError(f"{interpreter} doesn't exist") else: - interpreter = "python" + interpreter = cab.flavour.interpreter_binary - return [interpreter, "-u"] + args = cab.flavour.interpreter_command.format(python=interpreter).split() + + return args @dataclass @@ -67,6 +69,10 @@ class PythonCallableFlavour(_CallableFlavour): expected to be in the form of [package.]module.function """ kind: str = "python" + # name of python binary to use + interpreter_binary: str = "python" + # Full command used to launch interpreter. {python} gets substituted for the interpreter path + interpreter_command: str = "{python} -u" # don't log full command by default, as that's full of code log_full_command: bool = False @@ -155,6 +161,10 @@ class PythonCodeFlavour(_BaseFlavour): output_vars: bool = True # if True, command will have {}-substitutions done on it subst: bool = False + # name of python binary to use + interpreter_binary: str = "python" + # Full command used to launch interpreter. {python} gets substituted for the interpreter path + interpreter_command: str = "{python} -u" # don't log full command by default, as that's full of code log_full_command: bool = False diff --git a/stimela/commands/run.py b/stimela/commands/run.py index e786f715..d5d1bb40 100644 --- a/stimela/commands/run.py +++ b/stimela/commands/run.py @@ -107,9 +107,10 @@ def load_recipe_files(filenames: List[str]): full_deps.update(deps) # warn user if any includes failed - if full_deps.fails: - logger().warning(f"{len(full_deps.fails)} optional includes were not found, some cabs may not be available") - for path, dep in full_deps.fails.items(): + failed_includes = {path: dep for path, dep in full_deps.fails.items() if dep.warn} + if failed_includes: + logger().warning(f"{len(failed_includes)} optional includes were not found") + for path, dep in failed_includes.items(): logger().warning(f" {path} (from {dep.origin})") # merge into full config dependencies diff --git a/stimela/config.py b/stimela/config.py index 84c1353e..da212be0 100644 --- a/stimela/config.py +++ b/stimela/config.py @@ -6,6 +6,7 @@ from omegaconf.omegaconf import MISSING, OmegaConf from omegaconf.errors import OmegaConfBaseException from collections import OrderedDict +import psutil from yaml.error import YAMLError import stimela @@ -268,9 +269,12 @@ def _load(conf, config_file): runtime = dict( date=_ds, time=_ts, datetime=f"{_ds}-{_ts}", + ncpu=psutil.cpu_count(logical=True), node=platform.node().split('.', 1)[0], hostname=platform.node(), env={key: value.replace('${', '\${') for key, value in os.environ.items()}) + runtime['ncpu-logical'] = psutil.cpu_count(logical=True) + runtime['ncpu-physical'] = psutil.cpu_count(logical=False) conf.run = OmegaConf.create(runtime)