[HWORKS-135] Models backend should store metadata in tables instead of opensearch (logicalclocks#202)

Author: robzor92

python/hsml/engine/hopsworks_engine.py

@@ -16,22 +16,22 @@
 
 import os
 
-from hsml.core import native_hdfs_api
+from hsml.core import native_hdfs_api, model_api
 from hsml import client
 
 
 class HopsworksEngine:
     def __init__(self):
         self._native_hdfs_api = native_hdfs_api.NativeHdfsApi()
+        self._model_api = model_api.ModelApi()
 
     def mkdir(self, remote_path: str):
         remote_path = self._prepend_project_path(remote_path)
         self._native_hdfs_api.mkdir(remote_path)
         self._native_hdfs_api.chmod(remote_path, "ug+rwx")
 
-    def delete(self, remote_path: str):
-        remote_path = self._prepend_project_path(remote_path)
-        self._native_hdfs_api.rm(remote_path)
+    def delete(self, model_instance):
+        self._model_api.delete(model_instance)
 
     def upload(self, local_path: str, remote_path: str):
         local_path = self._get_abs_path(local_path)

python/hsml/engine/local_engine.py

@@ -16,21 +16,21 @@
 
 import os
 
-from hsml.core import dataset_api
+from hsml.core import dataset_api, model_api
 from hsml import client
 
 
 class LocalEngine:
     def __init__(self):
         self._dataset_api = dataset_api.DatasetApi()
+        self._model_api = model_api.ModelApi()
 
     def mkdir(self, remote_path: str):
         remote_path = self._prepend_project_path(remote_path)
         self._dataset_api.mkdir(remote_path)
 
-    def delete(self, remote_path: str):
-        remote_path = self._prepend_project_path(remote_path)
-        self._dataset_api.rm(remote_path)
+    def delete(self, model_instance):
+        self._model_api.delete(model_instance)
 
     def upload(self, local_path: str, remote_path: str):
         local_path = self._get_abs_path(local_path)

python/hsml/engine/model_engine.py

@@ -18,15 +18,12 @@
 import tempfile
 import uuid
 import time
-import importlib
 import os
-
+import importlib
 from tqdm.auto import tqdm
 
 from hsml.client.exceptions import RestAPIError, ModelRegistryException
-
 from hsml import client, util, constants
-
 from hsml.core import model_api, dataset_api
 
 from hsml.engine import local_engine, hopsworks_engine
@@ -493,7 +490,7 @@ def read_json(self, model_instance, resource):
                     tmp_dir.cleanup()
 
     def delete(self, model_instance):
-        self._engine.delete(model_instance.version_path)
+        self._engine.delete(model_instance)
 
     def set_tag(self, model_instance, name, value):
         """Attach a name/value tag to a model."""

python/hsml/model.py

@@ -38,6 +38,7 @@ def __init__(
         name,
         version=None,
         created=None,
+        creator=None,
         environment=None,
         description=None,
         experiment_id=None,
@@ -51,6 +52,8 @@ def __init__(
         input_example=None,
         framework=None,
         model_registry_id=None,
+        tags=None,
+        href=None,
         **kwargs,
     ):
         self._id = id
@@ -63,6 +66,7 @@ def __init__(
             self._description = description
 
         self._created = created
+        self._creator = creator
         self._environment = environment
         self._experiment_id = experiment_id
         self._project_name = project_name
@@ -267,6 +271,15 @@ def created(self):
     def created(self, created):
         self._created = created
 
+    @property
+    def creator(self):
+        """Creator of the model."""
+        return self._creator
+
+    @creator.setter
+    def creator(self, creator):
+        self._creator = creator
+
     @property
     def environment(self):
         """Input example of the model."""

python/hsml/python/model.py

@@ -28,6 +28,7 @@ def __init__(
         name,
         version=None,
         created=None,
+        creator=None,
         environment=None,
         description=None,
         experiment_id=None,
@@ -40,13 +41,16 @@ def __init__(
         training_dataset=None,
         input_example=None,
         model_registry_id=None,
+        tags=None,
+        href=None,
         **kwargs,
     ):
         super().__init__(
             id,
             name,
             version=version,
             created=created,
+            creator=creator,
             environment=environment,
             description=description,
             experiment_id=experiment_id,

python/hsml/sklearn/model.py

@@ -28,6 +28,7 @@ def __init__(
         name,
         version=None,
         created=None,
+        creator=None,
         environment=None,
         description=None,
         experiment_id=None,
@@ -40,13 +41,16 @@ def __init__(
         training_dataset=None,
         input_example=None,
         model_registry_id=None,
+        tags=None,
+        href=None,
         **kwargs,
     ):
         super().__init__(
             id,
             name,
             version=version,
             created=created,
+            creator=creator,
             environment=environment,
             description=description,
             experiment_id=experiment_id,

python/hsml/tensorflow/model.py

@@ -28,6 +28,7 @@ def __init__(
         name,
         version=None,
         created=None,
+        creator=None,
         environment=None,
         description=None,
         experiment_id=None,
@@ -40,13 +41,16 @@ def __init__(
         training_dataset=None,
         input_example=None,
         model_registry_id=None,
+        tags=None,
+        href=None,
         **kwargs,
     ):
         super().__init__(
             id,
             name,
             version=version,
             created=created,
+            creator=creator,
             environment=environment,
             description=description,
             experiment_id=experiment_id,

python/hsml/torch/model.py

@@ -28,6 +28,7 @@ def __init__(
         name,
         version=None,
         created=None,
+        creator=None,
         environment=None,
         description=None,
         experiment_id=None,
@@ -40,13 +41,16 @@ def __init__(
         training_dataset=None,
         input_example=None,
         model_registry_id=None,
+        tags=None,
+        href=None,
         **kwargs,
     ):
         super().__init__(
             id,
             name,
             version=version,
             created=created,
+            creator=creator,
             environment=environment,
             description=description,
             experiment_id=experiment_id,