Merge pull request #929 from mindsdb/staging

Release 22.7.2.0

Author: paxcema

lightwood/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'lightwood'
 __package_name__ = 'lightwood'
-__version__ = '22.6.1.2'
+__version__ = '22.7.2.0'
 __description__ = "Lightwood is a toolkit for automatic machine learning model building"
 __email__ = "community@mindsdb.com"
 __author__ = 'MindsDB Inc'

lightwood/analysis/explain.py

@@ -37,6 +37,7 @@ def explain(data: pd.DataFrame,
     # Setup base insights
     # ------------------------- #
     data = data.reset_index(drop=True)
+    predictions = predictions.reset_index(drop=True)
 
     row_insights = pd.DataFrame()
     global_insights = {}

lightwood/analysis/helpers/conf_stats.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import Dict
 from types import SimpleNamespace
 
@@ -55,10 +56,10 @@ def _get_stats(self, confs, preds, data, target, task_type='categorical'):
         mce: maximum value in `bins`.
         global_score: 1.0 minus absolute difference between accuracy and confidence over the entire validation set.
         """
-
+        confs = deepcopy(confs).reset_index(drop=True)
+        sorted_preds = deepcopy(preds).reset_index(drop=True)
+        sorted_inp = deepcopy(data).reset_index(drop=True)
         sorted_val = confs.sort_values(by='confidence', kind='stable')
-        sorted_preds = preds.reindex(sorted_val.index)
-        sorted_inp = data.reindex(sorted_val.index)
         sorted_inp['__mdb_confidence'] = sorted_val['confidence']
 
         if task_type == 'categorical':

lightwood/analysis/nc/calibrate.py

@@ -90,8 +90,11 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
             icp = icp_class(nc, cal_size=self.validation_size)
 
             output['icp']['__default'] = icp
+            icp_df = deepcopy(ns.data)
 
             # setup prediction cache to avoid additional .predict() calls
+            pred_is_list = isinstance(ns.normal_predictions['prediction'], list) and \
+                isinstance(ns.normal_predictions['prediction'][0], list)
             if ns.is_classification:
                 if ns.predictor.supports_proba:
                     icp.nc_function.model.prediction_cache = ns.normal_predictions[all_cat_cols].values
@@ -105,7 +108,7 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
                     predicted_classes = pd.get_dummies(preds).values  # inflate to one-hot enc
                     icp.nc_function.model.prediction_cache = predicted_classes
 
-            elif ns.is_multi_ts:
+            elif ns.is_multi_ts or pred_is_list:
                 # we fit ICPs for time series confidence bounds only at t+1 forecast
                 icp.nc_function.model.prediction_cache = np.array([p[0] for p in ns.normal_predictions['prediction']])
             else:
@@ -116,6 +119,9 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
 
             # fit additional ICPs in time series tasks with grouped columns
             if ns.tss.is_timeseries and ns.tss.group_by:
+                # generate a multiindex
+                midx = pd.MultiIndex.from_frame(icp_df[[*ns.tss.group_by, f'__mdb_original_{ns.tss.order_by[0]}']])
+                icp_df.index = midx
 
                 # create an ICP for each possible group
                 group_info = ns.data[ns.tss.group_by].to_dict('list')
@@ -127,7 +133,6 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
                     output['icp'][tuple(combination)] = deepcopy(icp)
 
             # calibrate ICP
-            icp_df = deepcopy(ns.data)
             icp_df, y = clean_df(icp_df, ns.target, ns.is_classification, output.get('label_encoders', None))
             output['icp']['__default'].index = icp_df.columns
             output['icp']['__default'].calibrate(icp_df.values, y)
@@ -137,11 +142,11 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
                 icp_df, icp, ns.dtype_dict[ns.target],
                 output, positive_domain=self.positive_domain, significance=self.fixed_significance)
             if not ns.is_classification:
-                result_df = pd.DataFrame(index=ns.data.index, columns=['confidence', 'lower', 'upper'], dtype=float)
+                result_df = pd.DataFrame(index=icp_df.index, columns=['confidence', 'lower', 'upper'], dtype=float)
                 result_df.loc[icp_df.index, 'lower'] = ranges[:, 0]
                 result_df.loc[icp_df.index, 'upper'] = ranges[:, 1]
             else:
-                result_df = pd.DataFrame(index=ns.data.index, columns=['confidence'], dtype=float)
+                result_df = pd.DataFrame(index=icp_df.index, columns=['confidence'], dtype=float)
 
             result_df.loc[icp_df.index, 'confidence'] = conf
 
@@ -152,10 +157,12 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
 
                 # add all predictions to DF
                 icps_df = deepcopy(ns.data)
-                if ns.is_multi_ts:
-                    icps_df[f'__predicted_{ns.target}'] = [p[0] for p in ns.normal_predictions['prediction']]
+                midx = pd.MultiIndex.from_frame(icps_df[[*ns.tss.group_by, f'__mdb_original_{ns.tss.order_by[0]}']])
+                icps_df.index = midx
+                if ns.is_multi_ts or pred_is_list:
+                    icps_df[f'__predicted_{ns.target}'] = np.array([p[0] for p in ns.normal_predictions['prediction']])
                 else:
-                    icps_df[f'__predicted_{ns.target}'] = ns.normal_predictions['prediction']
+                    icps_df[f'__predicted_{ns.target}'] = np.array(ns.normal_predictions['prediction'])
 
                 for group in icps['__mdb_groups']:
                     icp_df = icps_df
@@ -207,6 +214,7 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
             # consolidate all groups here
             output['icp']['__mdb_active'] = True
 
+        result_df.index = ns.data.index
         output['result_df'] = result_df
 
         info = {**info, **output}
@@ -216,12 +224,21 @@ def explain(self, row_insights: pd.DataFrame, global_insights: Dict[str, object]
                 **kwargs) -> Tuple[pd.DataFrame, Dict[str, object]]:
         ns = SimpleNamespace(**kwargs)
 
+        if 'confidence' in ns.predictions.columns:
+            # bypass calibrator if model already outputs confidence
+            row_insights['prediction'] = ns.predictions['prediction']
+            row_insights['confidence'] = ns.predictions['confidence']
+            if 'upper' in ns.predictions.columns and 'lower' in ns.predictions.columns:
+                row_insights['upper'] = ns.predictions['upper']
+                row_insights['lower'] = ns.predictions['lower']
+            return row_insights, global_insights
+
         if ns.analysis['icp']['__mdb_active']:
             icp_X = deepcopy(ns.data)
 
             # replace observed data w/predictions
             preds = ns.predictions['prediction']
-            if ns.tss.is_timeseries and ns.tss.horizon > 1:
+            if ns.tss.is_timeseries and (ns.tss.horizon > 1 or isinstance(preds[0], list)):
                 preds = [p[0] for p in preds]
 
                 for col in [f'timestep_{i}' for i in range(1, ns.tss.horizon)]:

lightwood/api/json_ai.py

@@ -220,27 +220,46 @@ def generate_json_ai(
     ):
         is_target_predicting_encoder = True
 
+    submodels = []
     if is_target_predicting_encoder:
-        submodels = [
-            {
-                "module": "Unit",
-                "args": {
-                    "target_encoder": "$encoders[self.target]",
-                    "stop_after": "$problem_definition.seconds_per_mixer",
-                },
-            }
-        ]
+        submodels.extend(
+            [
+                {
+                    "module": "Unit",
+                    "args": {
+                        "target_encoder": "$encoders[self.target]",
+                        "stop_after": "$problem_definition.seconds_per_mixer",
+                    },
+                }
+            ]
+        )
     else:
-        submodels = [
-            {
-                "module": "Neural",
-                "args": {
-                    "fit_on_dev": True,
-                    "stop_after": "$problem_definition.seconds_per_mixer",
-                    "search_hyperparameters": True,
-                },
-            }
-        ]
+        if not tss.is_timeseries:
+            submodels.extend(
+                [
+                    {
+                        "module": "Neural",
+                        "args": {
+                            "fit_on_dev": True,
+                            "stop_after": "$problem_definition.seconds_per_mixer",
+                            "search_hyperparameters": True,
+                        },
+                    }
+                ]
+            )
+        else:
+            submodels.extend(
+                [
+                    {
+                        "module": "NeuralTs",
+                        "args": {
+                            "fit_on_dev": True,
+                            "stop_after": "$problem_definition.seconds_per_mixer",
+                            "search_hyperparameters": True,
+                        },
+                    }
+                ]
+            )
 
         if (not tss.is_timeseries or tss.horizon == 1) and dtype_dict[target] not in (dtype.num_array, dtype.cat_array):
             submodels.extend(
@@ -268,7 +287,8 @@ def generate_json_ai(
                         "args": {
                             "fit_on_dev": True,
                             "stop_after": "$problem_definition.seconds_per_mixer",
-                            "horizon": "$problem_definition.timeseries_settings.horizon",
+                            "ts_analysis": "$ts_analysis",
+                            "tss": "$problem_definition.timeseries_settings",
                         },
                     }
                 ]
@@ -494,29 +514,30 @@ def _add_implicit_values(json_ai: JsonAI) -> JsonAI:
     is_ts = tss.is_timeseries
 
     # Add implicit arguments
-    # @TODO: Consider removing once we have a proper editor in studio
     mixers = json_ai.model['args']['submodels']
     for i in range(len(mixers)):
         if mixers[i]["module"] == "Unit":
             pass
 
-        elif mixers[i]["module"] == "Neural":
+        elif mixers[i]["module"] in ("Neural", "NeuralTs"):
             mixers[i]["args"]["target_encoder"] = mixers[i]["args"].get(
                 "target_encoder", "$encoders[self.target]"
             )
             mixers[i]["args"]["target"] = mixers[i]["args"].get("target", "$target")
             mixers[i]["args"]["dtype_dict"] = mixers[i]["args"].get(
                 "dtype_dict", "$dtype_dict"
             )
-            mixers[i]["args"]["timeseries_settings"] = mixers[i]["args"].get(
-                "timeseries_settings", "$problem_definition.timeseries_settings"
-            )
             mixers[i]["args"]["net"] = mixers[i]["args"].get(
                 "net",
                 '"DefaultNet"'
                 if not tss.is_timeseries or not tss.use_previous_target
                 else '"ArNet"',
             )
+            if mixers[i]["module"] == "NeuralTs":
+                mixers[i]["args"]["timeseries_settings"] = mixers[i]["args"].get(
+                    "timeseries_settings", "$problem_definition.timeseries_settings"
+                )
+                mixers[i]["args"]["ts_analysis"] = mixers[i]["args"].get("ts_analysis", "$ts_analysis")
 
         elif mixers[i]["module"] == "LightGBM":
             mixers[i]["args"]["target"] = mixers[i]["args"].get("target", "$target")
@@ -551,8 +572,17 @@ def _add_implicit_values(json_ai: JsonAI) -> JsonAI:
             mixers[i]["args"]["target_encoder"] = mixers[i]["args"].get(
                 "target_encoder", "$encoders[self.target]"
             )
-            if "horizon" not in mixers[i]["args"]:
-                mixers[i]["args"]["horizon"] = "$problem_definition.timeseries_settings.horizon"
+            mixers[i]["args"]["tss"] = mixers[i]["args"].get("tss", "$problem_definition.timeseries_settings")
+            mixers[i]["args"]["ts_analysis"] = mixers[i]["args"].get("ts_analysis", "$ts_analysis")
+            mixers[i]["args"]["fit_on_dev"] = mixers[i]["args"].get("fit_on_dev", "True")
+
+        elif mixers[i]["module"] == "NHitsMixer":
+            mixers[i]["args"]["target"] = mixers[i]["args"].get("target", "$target")
+            mixers[i]["args"]["horizon"] = "$problem_definition.timeseries_settings.horizon"
+            mixers[i]["args"]["ts_analysis"] = mixers[i]["args"].get(
+                "ts_analysis", "$ts_analysis"
+            )
+            problem_definition.fit_on_all = False  # takes too long otherwise
 
         elif mixers[i]["module"] in ("SkTime", "ProphetMixer"):
             mixers[i]["args"]["target"] = mixers[i]["args"].get("target", "$target")
@@ -666,6 +696,7 @@ def _add_implicit_values(json_ai: JsonAI) -> JsonAI:
                 "dtype_dict": "$dtype_dict",
                 "target": "$target",
                 "mode": "$mode",
+                "ts_analysis": "$ts_analysis"
             },
         },
         "timeseries_analyzer": {
@@ -807,11 +838,6 @@ def code_from_json_ai(json_ai: JsonAI) -> str:
 
 # Time-series blocks
 {ts_transform_code}
-"""
-    if ts_analyze_code is not None:
-        clean_body += f"""
-if self.mode != 'predict':
-{align(ts_analyze_code,1)}
 """
 
     clean_body += '\nreturn data'
@@ -835,12 +861,19 @@ def code_from_json_ai(json_ai: JsonAI) -> str:
     # Prepare features Body
     # ----------------- #
 
-    prepare_body = f"""
+    prepare_body = """
 self.mode = 'train'
 
 if self.statistical_analysis is None:
     raise Exception("Please run analyze_data first")
+"""
+    if ts_analyze_code is not None:
+        prepare_body += f"""
+if self.mode != 'predict':
+    {align(ts_analyze_code, 1)}
+"""
 
+    prepare_body += f"""
 # Column to encoder mapping
 self.encoders = {inline_dict(encoder_dict)}
 
@@ -1133,6 +1166,7 @@ def __init__(self):
 
         # Initial stats analysis
         self.statistical_analysis = None
+        self.ts_analysis = None
         self.runtime_log = dict()
 
     @timed

lightwood/api/types.py

@@ -73,6 +73,7 @@ class StatisticalAnalysis:
     :param bias:
     :param avg_words_per_sentence:
     :param positive_domain:
+    :param ts_stats:
     """ # noqa
 
     nr_rows: int
@@ -87,6 +88,7 @@ class StatisticalAnalysis:
     bias: object
     avg_words_per_sentence: object
     positive_domain: bool
+    ts_stats: dict
 
 
 @dataclass_json

lightwood/data/cleaner.py

@@ -147,13 +147,15 @@ def _standardize_datetime(element: object) -> Optional[float]:
     """
     Parses an expected date-time element. Intakes an element that can in theory be anything.
     """
+    if element is None or pd.isna(element):
+        return 0.0  # correct? TODO: Remove if the TS encoder can handle `None`
     try:
         date = parse_dt(str(element))
     except Exception:
         try:
             date = datetime.datetime.utcfromtimestamp(element)
         except Exception:
-            return None
+            return 0.0
 
     return date.timestamp()
 

lightwood/data/encoded_ds.py

@@ -140,13 +140,15 @@ class ConcatedEncodedDs(EncodedDs):
     """
     `ConcatedEncodedDs` abstracts over multiple encoded datasources (`EncodedDs`) as if they were a single entity.
     """  # noqa
+    # TODO: We should probably delete this abstraction, it's not really useful and it adds complexity/overhead
     def __init__(self, encoded_ds_arr: List[EncodedDs]) -> None:
         # @TODO: missing super() call here?
         self.encoded_ds_arr = encoded_ds_arr
         self.encoded_ds_lenghts = [len(x) for x in self.encoded_ds_arr]
         self.encoders = self.encoded_ds_arr[0].encoders
         self.encoder_spans = self.encoded_ds_arr[0].encoder_spans
         self.target = self.encoded_ds_arr[0].target
+        self.data_frame = pd.concat([x.data_frame for x in self.encoded_ds_arr])
 
     def __len__(self):
         """
@@ -166,17 +168,6 @@ def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]:
                 idx -= length
         raise StopIteration()
 
-    @property
-    def data_frame(self) -> pd.DataFrame:
-        """
-        Property that concatenates all underlying `EncodedDs`'s dataframes and returns them.
-        
-        Note: be careful to not modify a `ConcatedEncodedDs`, as you can see in the source, it will not have an effect.
-        
-        :return: Dataframe with all original data.
-        """  # noqa
-        return pd.concat([x.data_frame for x in self.encoded_ds_arr])
-
     def get_column_original_data(self, column_name: str) -> pd.Series:
         """
         See `lightwood.data.encoded_ds.EncodedDs.get_column_original_data()`.