Merge pull request #1182 from mindsdb/staging

Release 23.8.1.0

Author: paxcema

.flake8

@@ -1,4 +1,4 @@
 [flake8]
 max-line-length = 120
-ignore = E275,E402,F821,W503,W504,C408,W391
+ignore = E275,E402,F821,W503,W504,C408,W391,E721
 exclude = .git,__pycache__,docs,docssrc

.github/workflows/ligthtwood.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.8','3.9']
+        python-version: ["3.8","3.9","3.10","3.11"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

lightwood/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'lightwood'
 __package_name__ = 'lightwood'
-__version__ = '23.7.1.0'
+__version__ = '23.8.1.0'
 __description__ = "Lightwood is a toolkit for automatic machine learning model building"
 __email__ = "community@mindsdb.com"
 __author__ = 'MindsDB Inc'

lightwood/analysis/explain.py

@@ -4,12 +4,13 @@
 
 from dataprep_ml import StatisticalAnalysis
 
-from lightwood.helpers.log import log
+from lightwood.helpers.log import log, timed
 from lightwood.api.types import ProblemDefinition, PredictionArguments
 from lightwood.helpers.ts import get_inferred_timestamps
 from lightwood.analysis.base import BaseAnalysisBlock
 
 
+@timed
 def explain(data: pd.DataFrame,
             encoded_data: torch.Tensor,
             predictions: pd.DataFrame,

lightwood/analysis/nc/calibrate.py

@@ -200,10 +200,11 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
                     # save relevant predictions in the caches, then calibrate the ICP
                     pred_cache = icp_df.pop(f'__predicted_{ns.target}').values
                     if ns.is_multi_ts and ns.is_classification:
-                        # output['label_encoders'].transform(preds.reshape(-1, 1))
                         pred_cache = output['label_encoders'].transform([[p[0] for p in pred_cache]])
                     elif ns.is_multi_ts:
                         pred_cache = np.array([np.array(p) for p in pred_cache])
+                    elif ns.is_classification:
+                        pred_cache = output['label_encoders'].transform(pred_cache.reshape(-1, 1))
 
                     icps[tuple(group)].nc_function.model.prediction_cache = pred_cache
                     icp_df, y = clean_df(icp_df, ns, output.get('label_encoders', None))

lightwood/api/high_level.py

@@ -1,5 +1,4 @@
 import os
-from types import ModuleType
 from typing import Union
 import dill
 import pandas as pd
@@ -8,12 +7,8 @@
 from type_infer.infer import infer_types
 from lightwood.api.predictor import PredictorInterface
 from lightwood.api.json_ai import generate_json_ai
-import tempfile
-from lightwood.api.json_ai import code_from_json_ai as _code_from_json_ai
-import importlib.util
+from lightwood.helpers.codegen import code_from_json_ai as _code_from_json_ai, _module_from_code, _predictor_from_code
 import sys
-import random
-import string
 import gc
 import time
 from lightwood.helpers.log import log
@@ -107,10 +102,7 @@ def predictor_from_code(code: str) -> PredictorInterface:
 
     :returns: A lightwood ``Predictor`` object
     """
-    module_name = ''.join(random.choices(string.ascii_uppercase + string.digits, k=12))
-    module_name += str(time.time()).replace('.', '')
-    predictor = _module_from_code(code, module_name).Predictor()
-    return predictor
+    return _predictor_from_code(code)
 
 
 def code_from_problem(df: pd.DataFrame, problem_definition: Union[ProblemDefinition, dict]) -> str:
@@ -162,31 +154,6 @@ def predictor_from_state(state_file: str, code: str = None) -> PredictorInterfac
     return predictor
 
 
-def _module_from_code(code: str, module_name: str) -> ModuleType:
-    """
-    Create a python module (containing the generated ``Predictor`` class) from the code. This is both a python object and an associated temporary file on your filesystem
-
-    :param code: The ``Predictor``'s code in text form
-    :param module_name: The name of the newly created module
-
-    :returns: A python module object
-    """ # noqa
-    dirname = tempfile.gettempdir()
-    filename = os.urandom(24).hex() + str(time.time()).replace('.', '') + '.py'
-    path = os.path.join(dirname, filename)
-    if 'LIGHTWOOD_DEV_SAVE_TO' in os.environ:
-        path = os.environ['LIGHTWOOD_DEV_SAVE_TO']
-
-    with open(path, 'wb') as fp:
-        fp.write(code.encode('utf-8'))
-        spec = importlib.util.spec_from_file_location(module_name, fp.name)
-        temp_module = importlib.util.module_from_spec(spec)
-        sys.modules[module_name] = temp_module
-        spec.loader.exec_module(temp_module)
-
-    return temp_module
-
-
 def predictor_from_json_ai(json_ai: JsonAI) -> PredictorInterface:
     """
     Creates a ready-to-train ``Predictor`` object based on the details you specified inside your JsonAI.