Merge branch 'main' of github.com:fenke/corebridge

README.md

@@ -3,8 +3,8 @@
 
 <!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
 
-This file will become your README and also the index of your
-documentation.
+This package provides functions and classes to run wodan style
+processing functions in the Stactics AICore environment.
 
 ## Install
 
@@ -14,10 +14,55 @@ pip install corebridge
 
 ## How to use
 
-Fill me in please! Don’t forget code examples:
+### Introduction
+
+#### Wodan processing functions
+
+Wodan is a proprietary backend service that applies high performance,
+custom analytical processing to timeseries data in the Whysor data and
+dashboarding environment.
+
+Each wodan module defines one function that operates as the entry point.
+The parameter annotations in this function definition are used to format
+data and retrieve parameters from the originating call to the wodan api.
+This function is called with data retrieved according to a specification
+and with additional parameters as annotated.
+
+A simple function might look like:
+
+``` {python}
+import numpy as np
+
+def multiply(data:np.ndarray, multiplier:float=1.0):
+    return data * multiplier
+    
+```
+
+#### AICore modules
+
+For AICore one defines a class, always named `Module` with a constructor
+`__init__` and a method `infer`.
+
+This package defines a baseclass to quickly construct a custom `Module`
+class that uses a wodan processor function inside the AICore system:
+
+``` {python}
+import numpy as np
+import corebridge
+
+def multiply(data:np.ndarray, multiplier:float=1.0):
+    return data * multiplier
+
+class Module(corebridge.aicorebridge.AICoreModule):
+    def __init__(self, save_dir, assets_dir, *args, **kwargs):
+        super().__init__(read, save_dir, assets_dir, *args, **kwargs)
+    
+```
+
+That’s it. Well, you can add parameters to `__init__` that can be used
+as hyperparameters and you could override `infer` for the same reason.
 
 ``` python
-1+1
 ```
 
     2

corebridge/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.1"
+__version__ = "0.1.4"

corebridge/aicorebridge.py

@@ -7,6 +7,7 @@
 import typing
 import logging
 import inspect
+import traceback
 import datetime
 import json
 import pandas as pd
@@ -31,29 +32,29 @@ class AICoreModule(): pass
 
 # %% ../nbs/01_aicorebridge.ipynb 8
 @patch
-def __init__(self:AICoreModule, 
+def __init__(self:AICoreModule,
              processor:typing.Callable, # data processing function
-             save_dir:str, # path where the module can keep files 
+             save_dir:str, # path where the module can keep files
              assets_dir:str,
              *args, **kwargs):
-    
+
     self.init_time = datetime.datetime.utcnow()
     self.save_dir   = save_dir
     self.assets_dir = assets_dir
     self._init_processor(processor)
 
     self.init_args = args
-    self.init_kwargs = kwargs
+    self.init_kwargs = {k:v for k,v in kwargs.items() if v is not None}
 
 
 
 # %% ../nbs/01_aicorebridge.ipynb 9
 @patch
 def _init_processor(
-        self:AICoreModule, 
+        self:AICoreModule,
         processor:typing.Callable):
     """Initializes processor related variables on self"""
-    
+
     self.processor = processor
     self.processor_signature = inspect.signature(self.processor)
     self.processor_params = dict(self.processor_signature.parameters)
@@ -70,6 +71,7 @@ def call_processor(self:AICoreModule, calldata, **callargs):
 # %% ../nbs/01_aicorebridge.ipynb 11
 @patch
 def infer(self:AICoreModule, data:dict, *_, **kwargs):
+    msg=[]
     try:
 
         msg=[
@@ -84,18 +86,18 @@ def infer(self:AICoreModule, data:dict, *_, **kwargs):
         msg.append(f"lastSeen: {lastSeen}, recordformat: {recordformat}, timezone: {timezone}")
 
         calldata = self.get_call_data(
-            data, 
+            data,
             recordformat=recordformat,
             timezone=timezone,
             reversed=reversed)
-        
+
         msg.append(f"calldata shape: {calldata.shape}")
 
-        callargs = self.get_callargs(**kwargs)
+        callargs = self.get_callargs(**kwargs, **self.directory_args)
 
         for arg, val in callargs.items():
             msg.append(f"{arg}: {val}")
-            
+
         result = self.call_processor(calldata, **callargs)
         msg.append(f"result shape: {result.shape}")
 
@@ -107,21 +109,24 @@ def infer(self:AICoreModule, data:dict, *_, **kwargs):
                 timezone=timezone,
                 reversed=reversed)
         }
+
     except Exception as err:
+        msg.append(''.join(traceback.format_exception(None, err, err.__traceback__)))
+        syslog.exception(f"Exception {str(err)} in infer()")
         return {
-            'msg': f"Unexpected {err=}, {type(err)=}",
+            'msg': msg,
             'data': []
         }
 
 
-# %% ../nbs/01_aicorebridge.ipynb 12
+# %% ../nbs/01_aicorebridge.ipynb 13
 @patch
 def get_callargs(self:AICoreModule, **kwargs):
     "Get arguments for the processor call"
 
     # Remove null / None values
     kwargs = {k:v for k,v in kwargs.items() if v is not None}
-    
+
     metadata = kwargs.pop('metadata', {}) # TODO: historic metadata
 
     return {
@@ -130,15 +135,15 @@ def get_callargs(self:AICoreModule, **kwargs):
     }
 
 
-# %% ../nbs/01_aicorebridge.ipynb 13
+# %% ../nbs/01_aicorebridge.ipynb 14
 @patch
 def get_call_data(
-        self:AICoreModule, 
-        data:dict, 
-        recordformat='records', 
-        timezone='UTC', 
+        self:AICoreModule,
+        data:dict,
+        recordformat='records',
+        timezone='UTC',
         reversed=False):
-    
+
     "Convert data to the processor signature"
 
     df = core.set_time_index_zone(core.timeseries_dataframe_from_datadict(
@@ -155,4 +160,4 @@ def get_call_data(
     else:
         df.index = (df.index - datetime.datetime(1970,1,1, tzinfo=datetime.timezone.utc)) / datetime.timedelta(seconds=1)
         return df.reset_index().to_numpy()
-        
+

nbs/index.ipynb

@@ -23,7 +23,8 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "This file will become your README and also the index of your documentation."
+    "This package provides functions and classes to run wodan style processing functions in the \n",
+    "Stactics AICore environment."
    ]
   },
   {
@@ -53,27 +54,52 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Fill me in please! Don't forget code examples:"
+    "### Introduction\n",
+    "\n",
+    "#### Wodan processing functions\n",
+    "\n",
+    "Wodan is a proprietary backend service that applies high performance, custom analytical processing to timeseries data in the Whysor data and dashboarding environment.\n",
+    "\n",
+    "Each wodan module defines one function that operates as the entry point. The parameter annotations in this function definition are used to format data and retrieve parameters from the originating call to the wodan api. This function is called with data retrieved according to a specification and with additional parameters as annotated.\n",
+    "\n",
+    "A simple function might look like:\n",
+    "\n",
+    "```{python} \n",
+    "import numpy as np\n",
+    "\n",
+    "def multiply(data:np.ndarray, multiplier:float=1.0):\n",
+    "    return data * multiplier\n",
+    "    \n",
+    "```\n",
+    "\n"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "2"
-      ]
-     },
-     "execution_count": null,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
    "source": [
-    "1+1"
+    "#### AICore modules\n",
+    "\n",
+    "For AICore one defines a class, always named `Module` with a constructor `__init__` and a method `infer`.\n",
+    "\n",
+    "This package defines a baseclass to quickly construct a custom `Module` class that uses a wodan processor function inside the AICore system:\n",
+    "\n",
+    "\n",
+    "\n",
+    "```{python} \n",
+    "import numpy as np\n",
+    "import corebridge\n",
+    "\n",
+    "def multiply(data:np.ndarray, multiplier:float=1.0):\n",
+    "    return data * multiplier\n",
+    "\n",
+    "class Module(corebridge.aicorebridge.AICoreModule):\n",
+    "    def __init__(self, save_dir, assets_dir, *args, **kwargs):\n",
+    "        super().__init__(multiply, save_dir, assets_dir, *args, **kwargs)\n",
+    "    \n",
+    "```\n",
+    "\n",
+    "That's it. Well, you can add parameters to `__init__` that can be used as hyperparameters and you could override `infer` for the same reason. "
    ]
   },
   {

settings.ini

@@ -5,7 +5,7 @@
 ### Python library ###
 repo = corebridge
 lib_name = %(repo)s
-version = 0.1.1
+version = 0.1.4
 min_python = 3.7
 license = apache2
 black_formatting = False