diff --git a/scabha/configuratt/resolvers.py b/scabha/configuratt/resolvers.py index 6f84e847..96e3d38a 100644 --- a/scabha/configuratt/resolvers.py +++ b/scabha/configuratt/resolvers.py @@ -226,7 +226,7 @@ def load_include_files(keyword): flags = {} # check for (location)filename.yaml or (location)/filename.yaml style - match= re.match("^\\((.+)\\)/?(.+)$", incl) + match = re.match("^\\((.+)\\)/?(.+)$", incl) if match: modulename, filename = match.groups() if modulename.startswith("."): diff --git a/stimela/commands/doc.py b/stimela/commands/doc.py index 3d0c90c7..51225aa3 100644 --- a/stimela/commands/doc.py +++ b/stimela/commands/doc.py @@ -9,14 +9,13 @@ from omegaconf import OmegaConf import stimela -from scabha import configuratt from stimela import logger, log_exception from stimela.main import cli from scabha.cargo import ParameterCategory from stimela.kitchen.recipe import Recipe from stimela.kitchen.cab import Cab -from stimela.config import ConfigExceptionTypes from stimela.exceptions import RecipeValidationError +from stimela.task_stats import destroy_progress_bar from .run import load_recipe_files @@ -39,8 +38,6 @@ def doc(what: List[str] = [], do_list=False, implicit=False, obscure=False, all= log = logger() top_tree = Tree(f"stimela doc {' '.join(what)}", guide_style="dim") - found_something = False - default_recipe = None if required: max_category = ParameterCategory.Required @@ -70,7 +67,7 @@ def load_recipe(name: str, section: Dict): files_to_load = [] names_to_document = [] for item in what: - if os.path.isfile(item) and os.path.splitext(item)[1].lower() in (".yml", ".yaml"): + if os.path.splitext(item)[1].lower() in (".yml", ".yaml"): files_to_load.append(item) log.info(f"will load recipe/config file '{item}'") else: @@ -80,49 +77,60 @@ def load_recipe(name: str, section: Dict): if files_to_load: load_recipe_files(files_to_load) + destroy_progress_bar() + + log.info(f"loaded {len(stimela.CONFIG.cabs)} cab definition(s) and {len(stimela.CONFIG.lib.recipes)} recipe(s)") + + if not stimela.CONFIG.lib.recipes and not stimela.CONFIG.cabs: + log.error(f"Nothing to document") + sys.exit(2) + + recipes_to_document = set() + cabs_to_document = set() + for item in names_to_document: recipe_names = fnmatch.filter(stimela.CONFIG.lib.recipes.keys(), item) cab_names = fnmatch.filter(stimela.CONFIG.cabs.keys(), item) if not recipe_names and not cab_names: - log.error(f"'{item}' does not match any files, recipes or cab names. Try -l/--list") + log.error(f"'{item}' does not match any recipe or cab names. Try -l/--list") sys.exit(2) + recipes_to_document.update(recipe_names) + cabs_to_document.update(cab_names) + + # if nothing was specified, and only one cab/only one recipe is defined, print that + if not names_to_document: + if len(stimela.CONFIG.lib.recipes) == 1 and not stimela.CONFIG.cabs: + recipes_to_document.update(stimela.CONFIG.lib.recipes.keys()) + elif len(stimela.CONFIG.cabs) == 1 and not stimela.CONFIG.lib.recipes: + cabs_to_document.update(stimela.CONFIG.cabs.keys()) - for name in recipe_names: + if recipes_to_document or cabs_to_document: + for name in recipes_to_document: recipe = load_recipe(name, stimela.CONFIG.lib.recipes[name]) tree = top_tree.add(f"Recipe: [bold]{name}[/bold]") recipe.rich_help(tree, max_category=max_category) - for name in cab_names: + for name in cabs_to_document: cab = Cab(**stimela.CONFIG.cabs[name]) cab.finalize(config=stimela.CONFIG) tree = top_tree.add(f"Cab: [bold]{name}[/bold]") cab.rich_help(tree, max_category=max_category) - found_something = True - - if do_list or (not found_something and not default_recipe): - + # list recipes and cabs -- also do this by default if nothing explicit was documented + if do_list or not (recipes_to_document or cabs_to_document): if stimela.CONFIG.lib.recipes: subtree = top_tree.add("Recipes:") table = Table.grid("", "", padding=(0,2)) for name, recipe in stimela.CONFIG.lib.recipes.items(): table.add_row(f"[bold]{name}[/bold]", recipe.info) subtree.add(table) - elif not do_list and not found_something: - log.error(f"nothing particular to document, please specify a recipe name or a cab name, or use -l/--list") - sys.exit(2) - if default_recipe and not found_something: - recipe = load_recipe(name, stimela.CONFIG.lib.recipes[default_recipe]) - tree = top_tree.add(f"Recipe: [bold]{default_recipe}[/bold]") - recipe.rich_help(tree, max_category=max_category) - - if do_list: - subtree = top_tree.add("Cabs:") - table = Table.grid("", "", padding=(0,2)) - for name, cab in stimela.CONFIG.cabs.items(): - table.add_row(f"[bold]{name}[/bold]", cab.info) - subtree.add(table) + if stimela.CONFIG.cabs: + subtree = top_tree.add("Cabs:") + table = Table.grid("", "", padding=(0,2)) + for name, cab in stimela.CONFIG.cabs.items(): + table.add_row(f"[bold]{name}[/bold]", cab.info) + subtree.add(table) rich_print(top_tree) diff --git a/stimela/commands/run.py b/stimela/commands/run.py index b66d5781..9832a681 100644 --- a/stimela/commands/run.py +++ b/stimela/commands/run.py @@ -5,7 +5,8 @@ import yaml import sys import traceback -import atexit +import re +import importlib from datetime import datetime from typing import List, Optional, Tuple from collections import OrderedDict @@ -30,6 +31,18 @@ def load_recipe_files(filenames: List[str]): full_conf = OmegaConf.create() full_deps = configuratt.ConfigDependencies() for filename in filenames: + # check for (location)filename.yaml or (location)/filename.yaml style + match1 = re.fullmatch("^\\((.+)\\)/?(.+)$", filename) + match2 = re.fullmatch("^([\w.]+)::(.+)$", filename) + if match1 or match2: + modulename, filename = (match1 or match2).groups() + try: + mod = importlib.import_module(modulename) + except ImportError as exc: + log_exception(f"error importing {modulename}", exc) + sys.exit(2) + filename = os.path.join(os.path.dirname(mod.__file__), filename) + # try loading try: conf, deps = configuratt.load(filename, use_sources=[stimela.CONFIG, full_conf], no_toplevel_cache=True) except ConfigExceptionTypes as exc: @@ -153,7 +166,7 @@ def convert_value(value): except Exception as exc: log_exception(f"error parsing {pp}", exc) errcode = 2 - elif os.path.isfile(pp) and os.path.splitext(pp)[1].lower() in (".yml", ".yaml"): + elif os.path.splitext(pp)[1].lower() in (".yml", ".yaml"): files_to_load.append(pp) log.info(f"will load recipe/config file '{pp}'") else: