Merge pull request #359 from caracal-pipeline/clickify-cleanup

Clickify cleanup

pyproject.toml

@@ -15,7 +15,6 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.9"
-numpy = "^1.20"
 munch = "^2.5.0"
 omegaconf = "^2.1"
 importlib_metadata = { version = "4.13.0", python = "3.7" }

scabha/evaluator.py

@@ -3,7 +3,6 @@
 import os.path
 import fnmatch
 import pyparsing
-import numpy as np
 pyparsing.ParserElement.enable_packrat()
 from pyparsing import *
 from pyparsing import common
@@ -182,7 +181,7 @@ def is_str(x):
 
     def IS_NUM(self, evaluator, args):
         def is_num(x):
-            return np.isscalar(x) and (np.isreal(x) or np.isscalar(x)) 
+            return isinstance(x, (bool, int, float, complex)) 
         return self.evaluate_generic_callable(evaluator, "IS_NUM", is_num, args, min_args=1, max_args=1)
 
     def IF(self, evaluator, args):

scabha/schema_utils.py

@@ -125,9 +125,9 @@ def nested_schema_to_dataclass(nested: Dict[str, Dict], class_name: str, bases=(
 
 _atomic_types = dict(bool=bool, str=str, int=int, float=float)
 
-def _validate_list(text: str, element_type, schema, sep=",", brackets=True, pass_missing_as_none=False):
+def _validate_list(text: str, element_type, schema, sep=",", brackets=True):
     if not text:
-        if schema.default in (UNSET, _UNSET_DEFAULT) and pass_missing_as_none:
+        if schema.default in (UNSET, _UNSET_DEFAULT):
             return None
         else:
             return schema.default
@@ -142,9 +142,9 @@ def _validate_list(text: str, element_type, schema, sep=",", brackets=True, pass
     except ValueError:
         raise click.BadParameter(f"can't convert to '{schema.dtype}'")
 
-def _validate_tuple(text: str, element_types, schema, sep=",", brackets=True, pass_missing_as_none=False):
+def _validate_tuple(text: str, element_types, schema, sep=",", brackets=True):
     if not text:
-        if schema.default in (UNSET, _UNSET_DEFAULT) and pass_missing_as_none:
+        if schema.default in (UNSET, _UNSET_DEFAULT):
             return None
         else:
             return schema.default
@@ -176,8 +176,13 @@ def clickify_parameters(schemas: Union[str, Dict[str, Any]],
     package.
 
     Args:
-        schemas (str): The YAML filename for the parameter schema.
-        default_policies: Needs documentation.
+        schemas (str or Dict): Either the YAML filename from which the parameter schema is loaded,
+            containing inputs, outputs and an [optional] policies section,
+            or a DictConfig object containing inputs/outputs/policies.
+            See https://stimela.readthedocs.io/en/latest/reference/schema_ref.html
+        default_policies: default policies applied to the schema, overrides the policies section
+            if supplied. See ParameterPolicies in scabha/cargo.py, and
+            https://stimela.readthedocs.io/en/latest/reference/policies.html  
 
     Example:
     =======
@@ -195,9 +200,6 @@ def clickify_parameters(schemas: Union[str, Dict[str, Any]],
         outputs:
         {}
 
-        policies:
-            pass_missing_as_none: true
-
     The corresponding python code uses clickify_parameters as 
     a decorator:
         import click
@@ -208,12 +210,9 @@ def clickify_parameters(schemas: Union[str, Dict[str, Any]],
 
         @click.command()
         @clickify_parameters('hello/hello.yml')
-        def hello(**kw):
-            # Simple program that greets NAME for a total of COUNT times.
-            opts = OmegaConf.create(kw)
-
-            for x in range(opts.count):
-                click.echo(f"Hello {opts.name}!")
+        def hello(count: int = 1, name: Optional[str] = None):
+            for x in range(count):
+                click.echo(f"Hello {name}!")
     Returns:
         Nothing
     """
@@ -254,9 +253,22 @@ def hello(**kw):
             name = name.replace("_", "-").replace(".", "-")
             optname = f"--{name}"
             dtype = schema.dtype
+            if type(dtype) is str:
+                dtype = dtype.strip()
             validator = None
             multiple = False
             nargs = 1
+            # process optional
+            optional_match = re.fullmatch(r"Optional\[(.*)\]", dtype)
+            if optional_match:
+                str_dtype = dtype = optional_match.group(1).strip()
+            else:
+                str_dtype = str(dtype)
+            metavar = schema.metavar or str_dtype
+            is_unset = schema.default in (UNSET, _UNSET_DEFAULT)
+            kwargs = dict()
+            if not is_unset and not schema.suppress_cli_default and nargs != -1:
+                kwargs['default'] = schema.default
 
             # sort out option type. Atomic type?
             if dtype in _atomic_types:
@@ -273,44 +285,56 @@ def hello(**kw):
                 if list_match:
                     elem_type = _atomic_types.get(list_match.group(1).strip(), str)
                     if policies.repeat == 'list':
+                        if not policies.positional:
+                            raise SchemaError(f"click parameter '{name}': repeat=list policy is only supported for positional=true parameters")
                         nargs = -1
                         dtype = elem_type
+                        metavar = schema.metavar or f"{elem_type.__name__} ..."
                     elif policies.repeat == 'repeat':
                         multiple = True
                         dtype = elem_type
+                        metavar = schema.metavar or f"{elem_type.__name__}"
+                        # multiple options have a click default of () not None
+                        if 'default' not in kwargs:
+                            kwargs['default'] = None
                     elif policies.repeat == '[]':  # else assume [X,Y] or X,Y syntax
                         dtype = str
                         validator = lambda ctx, param, value, etype=dtype, schema=schema, _type=elem_type: \
                             _validate_list(value, element_type=_type, schema=schema,
-                                           pass_missing_as_none=policies.pass_missing_as_none)
+                                           brackets=False)
+                        metavar = schema.metavar or f"{elem_type.__name__},{elem_type.__name__},..."
                     elif policies.repeat is not None:  # assume XrepY syntax
                         dtype = str
+                        sep = policies.repeat
                         validator = lambda ctx, param, value, etype=dtype, schema=schema, _type=elem_type: \
                             _validate_list(value, element_type=_type, schema=schema,
-                                           sep=policies.repeat, brackets=False,
-                                           pass_missing_as_none=policies.pass_missing_as_none)
+                                           sep=sep, brackets=False)
+                        metavar = schema.metavar or f"{elem_type.__name__}{sep}{elem_type.__name__}{sep}..."
                     else:
                         raise SchemaError(f"list-type parameter '{name}' does not have a repeat policy set")
                 elif tuple_match:
                     elem_types = tuple(_atomic_types.get(t.strip(), str) for t in tuple_match.group(1).split(","))
-                    if policies.repeat == 'list' or policies.repeat == 'repeat':
+                    if policies.repeat == 'list':
                         nargs = len(elem_types)
                         dtype = elem_types
+                        metavar = schema.metavar or " ".join(t.__name__ for t in elem_types)
+                    elif policies.repeat == 'repeat':
+                        raise SchemaError(f"tuple-type parameter '{name}' has unsupported repeat policy 'repeat'")
                     elif policies.repeat == '[]':  # else assume [X,Y] or X,Y syntax
                         dtype = str
+                        metavar = schema.metavar or ",".join((t.__name__ for t in elem_types))
                         validator = lambda ctx, param, value, etype=dtype, \
                                 schema=schema, _type=elem_types: \
                                 _validate_tuple(value, element_types=_type,
-                                                schema=schema,
-                                                pass_missing_as_none=policies.pass_missing_as_none)
+                                                schema=schema, brackets=False)
                     elif policies.repeat is not None:  # assume XrepY syntax
                         dtype = str
+                        metavar = schema.metavar or policies.repeat.join((t.__name__ for t in elem_types))
                         validator = lambda ctx, param, value, etype=dtype, \
                         schema=schema, _type=elem_types: \
                             _validate_tuple(value, element_types=_type,
                                             schema=schema,
-                                            sep=policies.repeat, brackets=False,
-                                            pass_missing_as_none=policies.pass_missing_as_none)
+                                            sep=policies.repeat, brackets=False)
                     else:
                         raise SchemaError(f"tuple-type parameter '{name}' does not have a repeat policy set")
                 else:
@@ -327,19 +351,13 @@ def hello(**kw):
                 optnames.append(f"-{schema.abbreviation}")
 
             if policies.positional:
-                kwargs = dict(type=dtype, callback=validator, required=schema.required, nargs=nargs,
-                              metavar=schema.metavar)
-                if not schema.default in (UNSET, _UNSET_DEFAULT) and not schema.suppress_cli_default:
-                    kwargs['default'] = schema.default
+                kwargs.update(type=dtype, callback=validator, required=schema.required, nargs=nargs,
+                              metavar=metavar)
                 deco = click.argument(name, **kwargs)
             else:
-                kwargs = dict(type=dtype, callback=validator,
+                kwargs.update(type=dtype, callback=validator, 
                               required=schema.required, multiple=multiple,
-                              metavar=schema.metavar, help=schema.info)
-                if not schema.default in (UNSET, _UNSET_DEFAULT) and not schema.suppress_cli_default:
-                    kwargs['default'] = schema.default
-                elif schema.default in (UNSET, _UNSET_DEFAULT) and policies.pass_missing_as_none:
-                    kwargs['default'] = None
+                              metavar=metavar, help=schema.info)
                 deco = click.option(*optnames, **kwargs)
             if decorator_chain is None:
                 decorator_chain = deco

stimela/kitchen/step.py

@@ -617,7 +617,7 @@ def run(self, backend: Optional[Dict] = None, subst: Optional[Dict[str, Any]] =
                 level = logging.WARNING if self.skip else logging.ERROR
                 if not exc.logged:
                     if type(exc) is SubstitutionErrorList:
-                        self.log_exception(StepValidationError(f"unresolved {{}}-substitution(s) in inputs:", exc.nested), severity=severity)
+                        self.log_exception(StepValidationError(f"unresolved {{}}-substitution(s) in outputs:", exc.nested), severity=severity)
                         # for err in exc.errors:
                         #     self.log.log(level, f"  {err}")
                     else:

tests/scabha_tests/test_clickify.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+from typing import Tuple, List, Optional
+import sys
+import os.path
+import click
+from scabha.schema_utils import clickify_parameters
+
+schema_file = os.path.join(os.path.dirname(__file__), "test_clickify.yaml")
+
+@click.command()
+@clickify_parameters(schema_file)
+def func(name: str, i: int, j: Optional[float] = 1,
+         remainder: Optional[List[str]] = None, 
+         k: float=2,
+         tup: Optional[Tuple[int, str]] = None, 
+         files1: Optional[List[str]] = None,
+         files2: Optional[List[str]] = None,
+         files3: Optional[List[str]] = None,
+         output: str = None):
+    print(f"name:{name} i:{i} j:{j} k:{k} tup:{tup}")
+    print(f"remainder: {remainder}")
+    print(f"files1: {files1}")
+    print(f"files2: {files2}")
+    print(f"files3: {files3}")
+    print(f"output: {output}")
+
+if __name__ == "__main__":
+    sys.exit(func())
+

tests/scabha_tests/test_clickify.yaml

@@ -0,0 +1,42 @@
+inputs:
+  name: 
+    dtype: str
+    required: true
+    policies: 
+      positional: true
+  i: 
+    dtype: int 
+    required: true
+    policies: 
+      positional: true
+  remainder:
+    dtype: List[str]
+    policies: 
+      positional: true
+      repeat: list
+  j: 
+    dtype: float
+  k: 
+    dtype: float
+    default: 2
+  tup: 
+    dtype: Optional[Tuple[int, str]] 
+  files1: 
+    dtype: List[str]
+    policies: 
+      repeat: ' '
+  files2: 
+    dtype: List[File]
+    required: false
+    policies: 
+      repeat: repeat
+  files3: 
+    dtype: List[File]
+    required: false
+    policies: 
+      repeat: '[]'
+
+outputs:
+  output:
+    dtype: File
+    required: false