[HWORKS-1037] Save model files in /Models/name/version/Files dir

Author: javierdlrm

python/hopsworks_common/constants.py

@@ -163,6 +163,8 @@ class MODEL:
 
 class MODEL_REGISTRY:
     HOPSFS_MOUNT_PREFIX = "/hopsfs/"
+    ARTIFACTS_DIR_NAME = "Artifacts"
+    MODEL_FILES_DIR_NAME = "Files"
 
 
 class MODEL_SERVING:

python/hsml/engine/model_engine.py

@@ -80,11 +80,11 @@ def _upload_additional_resources(self, model_instance):
         return model_instance
 
     def _copy_or_move_hopsfs_model_item(
-        self, item_attr, to_model_version_path, keep_original_files
+        self, item_attr, to_model_files_path, keep_original_files
     ):
         """Copy or move model item from a hdfs path to the model version folder in the Models dataset. It works with files and folders."""
         path = item_attr["path"]
-        to_hdfs_path = os.path.join(to_model_version_path, os.path.basename(path))
+        to_hdfs_path = os.path.join(to_model_files_path, os.path.basename(path))
         if keep_original_files:
             self._engine.copy(path, to_hdfs_path)
         else:
@@ -93,7 +93,7 @@ def _copy_or_move_hopsfs_model_item(
     def _copy_or_move_hopsfs_model(
         self,
         from_hdfs_model_path,
-        to_model_version_path,
+        to_model_files_path,
         keep_original_files,
         update_upload_progress,
     ):
@@ -122,7 +122,7 @@ def _copy_or_move_hopsfs_model(
             )["items"]:
                 path_attr = entry["attributes"]
                 self._copy_or_move_hopsfs_model_item(
-                    path_attr, to_model_version_path, keep_original_files
+                    path_attr, to_model_files_path, keep_original_files
                 )
                 if path_attr.get("dir", False):
                     n_dirs += 1
@@ -132,7 +132,7 @@ def _copy_or_move_hopsfs_model(
         else:
             # if path is a file, copy/move it
             self._copy_or_move_hopsfs_model_item(
-                model_path_attr, to_model_version_path, keep_original_files
+                model_path_attr, to_model_files_path, keep_original_files
             )
             n_files += 1
             update_upload_progress(n_dirs=n_dirs, n_files=n_files)
@@ -156,7 +156,7 @@ def _download_model_from_hopsfs_recursive(
 
             if path_attr.get("dir", False):
                 # otherwise, make a recursive call for the folder
-                if basename == "Artifacts":
+                if basename == constants.MODEL_REGISTRY.ARTIFACTS_DIR_NAME:
                     continue  # skip Artifacts subfolder
                 local_folder_path = os.path.join(to_local_path, basename)
                 os.mkdir(local_folder_path)
@@ -195,11 +195,11 @@ def _download_model_from_hopsfs(
     def _upload_local_model(
         self,
         from_local_model_path,
-        to_model_version_path,
+        to_model_files_path,
         update_upload_progress,
         upload_configuration=None,
     ):
-        """Copy or upload model files from a local path to the model version folder in the Models dataset."""
+        """Copy or upload model files from a local path to the model files folder in the Models dataset."""
         n_dirs, n_files = 0, 0
         if os.path.isdir(from_local_model_path):
             # if path is a dir, upload files and folders iteratively
@@ -210,7 +210,7 @@ def _upload_local_model(
                 # - files is the list of file names present in the root dir
                 # we need to replace the local path prefix with the hdfs path prefix (i.e., /srv/hops/....../root with /Projects/.../)
                 remote_base_path = root.replace(
-                    from_local_model_path, to_model_version_path
+                    from_local_model_path, to_model_files_path
                 )
                 for d_name in dirs:
                     self._engine.mkdir(remote_base_path + "/" + d_name)
@@ -228,7 +228,7 @@ def _upload_local_model(
             # if path is a file, upload file
             self._engine.upload(
                 from_local_model_path,
-                to_model_version_path,
+                to_model_files_path,
                 upload_configuration=upload_configuration,
             )
             n_files += 1
@@ -249,14 +249,14 @@ def _save_model_from_local_or_hopsfs_mount(
                 from_hdfs_model_path=model_path.replace(
                     constants.MODEL_REGISTRY.HOPSFS_MOUNT_PREFIX, ""
                 ),
-                to_model_version_path=model_instance.version_path,
+                to_model_files_path=model_instance.files_path,
                 keep_original_files=keep_original_files,
                 update_upload_progress=update_upload_progress,
             )
         else:
             self._upload_local_model(
                 from_local_model_path=model_path,
-                to_model_version_path=model_instance.version_path,
+                to_model_files_path=model_instance.files_path,
                 update_upload_progress=update_upload_progress,
                 upload_configuration=upload_configuration,
             )
@@ -365,6 +365,7 @@ def save(
                 if step["id"] == 0:
                     # Create folders
                     self._engine.mkdir(model_instance.version_path)
+                    self._engine.mkdir(model_instance.files_path)
                 if step["id"] == 1:
 
                     def update_upload_progress(n_dirs=0, n_files=0, step=step):
@@ -374,7 +375,7 @@ def update_upload_progress(n_dirs=0, n_files=0, step=step):
 
                     update_upload_progress(n_dirs=0, n_files=0)
 
-                    # Upload Model files from local path to /Models/{model_instance._name}/{model_instance._version}
+                    # Upload Model files from local path to /Models/{model_instance._name}/{model_instance._version}/Files
                     # check local absolute
                     if os.path.isabs(model_path) and os.path.exists(model_path):
                         self._save_model_from_local_or_hopsfs_mount(
@@ -401,7 +402,7 @@ def update_upload_progress(n_dirs=0, n_files=0, step=step):
                     ):  # check hdfs relative and absolute
                         self._copy_or_move_hopsfs_model(
                             from_hdfs_model_path=model_path,
-                            to_model_version_path=model_instance.version_path,
+                            to_model_files_path=model_instance.files_path,
                             keep_original_files=keep_original_files,
                             update_upload_progress=update_upload_progress,
                         )
@@ -446,7 +447,7 @@ def update_download_progress(n_dirs, n_files, done=False):
             )
 
         try:
-            from_hdfs_model_path = model_instance.version_path
+            from_hdfs_model_path = model_instance.files_path
             if from_hdfs_model_path.startswith("hdfs:/"):
                 projects_index = from_hdfs_model_path.find("/Projects", 0)
                 from_hdfs_model_path = from_hdfs_model_path[projects_index:]

python/hsml/engine/serving_engine.py

@@ -19,7 +19,7 @@
 import uuid
 from typing import Dict, List, Union
 
-from hsml import util
+from hsml import util, constants
 from hsml.client.exceptions import ModelServingException, RestAPIError
 from hsml.client.istio.utils.infer_type import InferInput
 from hsml.constants import (
@@ -322,7 +322,7 @@ def download_artifact(self, deployment_instance):
             str(uuid.uuid4()),
             deployment_instance.model_name,
             str(deployment_instance.model_version),
-            "Artifacts",
+            constants.MODEL_REGISTRY.ARTIFACTS_DIR_NAME,
         )
         to_artifact_version_path = (
             to_artifacts_path + "/" + str(deployment_instance.artifact_version)

python/hsml/model.py

@@ -22,7 +22,7 @@
 
 import humps
 from hopsworks_common import usage
-from hsml import client, util
+from hsml import client, util, constants
 from hsml.constants import ARTIFACT_VERSION
 from hsml.constants import INFERENCE_ENDPOINTS as IE
 from hsml.core import explicit_provenance
@@ -544,6 +544,11 @@ def model_path(self):
     def version_path(self):
         """path of the model including version folder. Resolves to /Projects/{project_name}/Models/{name}/{version}"""
         return "{}/{}".format(self.model_path, str(self.version))
+    
+    @property
+    def files_path(self):
+        """path of the model files including version and files folder. Resolves to /Projects/{project_name}/Models/{name}/{version}/Files"""
+        return "{}/{}/{}".format(self.model_path, str(self.version), constants.MODEL_REGISTRY.MODEL_FILES_DIR_NAME)
 
     @property
     def shared_registry_project_name(self):