[FSTORE-1464] Unify tags_api, inode and dataset_api (logicalclocks#240)

* Move tag, util, dataset_api and inode

* Separate part of util independent from hsfs

* Merge hopsworks/util into hopsworks_common

* Create alias for git_file_status

* Fix aliases

* Fix mistype in git_file_status

* Move tags_api

* Adapt the moved files

* Add alias for inode

* Rename FeatureStoreEncoder to Encoder

* Fix mocking

* Remove dataset_api from hopsworks_common

* Move dataset_api from hopsworks

* Adapt dataset_api

* Fix dataset_api

* Fix DatasetApi.__init__

* Fix test_python

Author: aversey

python/hopsworks/core/dataset_api.py

@@ -1,5 +1,5 @@
 #
-#   Copyright 2022 Logical Clocks AB
+#   Copyright 2024 Hopsworks AB
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -14,62 +14,11 @@
 #   limitations under the License.
 #
 
-import json
+from hopsworks_common.git_file_status import (
+    GitFileStatus,
+)
 
-import humps
-from hopsworks import util
 
-
-class GitFileStatus:
-    def __init__(
-        self,
-        file=None,
-        status=None,
-        extra=None,
-        **kwargs,
-    ):
-        self._file = file
-        self._status = status
-        self._extra = extra
-
-    @classmethod
-    def from_response_json(cls, json_dict):
-        json_decamelized = humps.decamelize(json_dict)
-        if "count" in json_decamelized:
-            if json_decamelized["count"] == 0:
-                return []
-            return [cls(**file_status) for file_status in json_decamelized["items"]]
-        else:
-            return cls(**json_decamelized)
-
-    @property
-    def file(self):
-        """Path to the file"""
-        return self._file
-
-    @property
-    def status(self):
-        """Status of the file
-        Unmodified         StatusCode = ' '
-        Untracked          StatusCode = '?'
-        Modified           StatusCode = 'M'
-        Added              StatusCode = 'A'
-        Deleted            StatusCode = 'D'
-        Renamed            StatusCode = 'R'
-        Copied             StatusCode = 'C'
-        UpdatedButUnmerged StatusCode = 'U'"""
-        return self._status
-
-    @property
-    def extra(self):
-        """Extra contains additional information, such as the previous name in a rename"""
-        return self._extra
-
-    def json(self):
-        return json.dumps(self, cls=util.Encoder)
-
-    def __str__(self):
-        return self.json()
-
-    def __repr__(self):
-        return f"GitFileStatus({self._file!r}, {self._status!r}, {self._extra!r})"
+__all__ = [
+    GitFileStatus,
+]

python/hopsworks/git_file_status.py

@@ -1,5 +1,5 @@
 #
-#   Copyright 2022 Logical Clocks AB
+#   Copyright 2024 Hopsworks AB
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -14,75 +14,75 @@
 #   limitations under the License.
 #
 
-from json import JSONEncoder
-from urllib.parse import urljoin, urlparse
-
-from hopsworks import client
-from hopsworks.client.exceptions import JobException
-from hopsworks.git_file_status import GitFileStatus
-
-
-class Encoder(JSONEncoder):
-    def default(self, obj):
-        try:
-            return obj.to_dict()
-        except AttributeError:
-            return super().default(obj)
-
-
-def convert_to_abs(path, current_proj_name):
-    abs_project_prefix = "/Projects/{}".format(current_proj_name)
-    if not path.startswith(abs_project_prefix):
-        return abs_project_prefix + "/" + path
-    else:
-        return path
-
-
-def validate_job_conf(config, project_name):
-    # User is required to set the appPath programmatically after getting the configuration
-    if (
-        config["type"] != "dockerJobConfiguration"
-        and config["type"] != "flinkJobConfiguration"
-        and "appPath" not in config
-    ):
-        raise JobException("'appPath' not set in job configuration")
-    elif "appPath" in config and not config["appPath"].startswith("hdfs://"):
-        config["appPath"] = "hdfs://" + convert_to_abs(config["appPath"], project_name)
-
-    # If PYSPARK application set the mainClass, if SPARK validate there is a mainClass set
-    if config["type"] == "sparkJobConfiguration":
-        if config["appPath"].endswith(".py"):
-            config["mainClass"] = "org.apache.spark.deploy.PythonRunner"
-        elif "mainClass" not in config:
-            raise JobException("'mainClass' not set in job configuration")
-
-    return config
-
-
-def convert_git_status_to_files(files):
-    # Convert GitFileStatus to list of file paths
-    if isinstance(files[0], GitFileStatus):
-        tmp_files = []
-        for file_status in files:
-            tmp_files.append(file_status.file)
-        files = tmp_files
-
-    return files
-
-
-def get_hostname_replaced_url(sub_path: str):
-    """
-    construct and return an url with public hopsworks hostname and sub path
-    :param self:
-    :param sub_path: url sub-path after base url
-    :return: href url
-    """
-    href = urljoin(client.get_instance()._base_url, sub_path)
-    url_parsed = client.get_instance().replace_public_host(urlparse(href))
-    return url_parsed.geturl()
-
-
-def is_interactive():
-    import __main__ as main
-
-    return not hasattr(main, "__file__")
+from hopsworks_common.util import (
+    FEATURE_STORE_NAME_SUFFIX,
+    VALID_EMBEDDING_TYPE,
+    Encoder,
+    FeatureGroupWarning,
+    JobWarning,
+    StatisticsWarning,
+    StorageWarning,
+    ValidationWarning,
+    VersionWarning,
+    _loading_animation,
+    append_feature_store_suffix,
+    autofix_feature_name,
+    check_timestamp_format_from_date_string,
+    convert_event_time_to_timestamp,
+    convert_git_status_to_files,
+    convert_to_abs,
+    feature_group_name,
+    get_dataset_type,
+    get_delta_datestr_from_timestamp,
+    get_feature_group_url,
+    get_host_name,
+    get_hostname_replaced_url,
+    get_hudi_datestr_from_timestamp,
+    get_job_url,
+    get_timestamp_from_date_string,
+    is_interactive,
+    is_runtime_notebook,
+    run_with_loading_animation,
+    setup_pydoop,
+    strip_feature_store_suffix,
+    validate_embedding_feature_type,
+    validate_job_conf,
+    verify_attribute_key_names,
+)
+
+
+__all__ = [
+    FEATURE_STORE_NAME_SUFFIX,
+    VALID_EMBEDDING_TYPE,
+    Encoder,
+    FeatureGroupWarning,
+    JobWarning,
+    StatisticsWarning,
+    StorageWarning,
+    ValidationWarning,
+    VersionWarning,
+    _loading_animation,
+    append_feature_store_suffix,
+    autofix_feature_name,
+    check_timestamp_format_from_date_string,
+    convert_event_time_to_timestamp,
+    convert_git_status_to_files,
+    convert_to_abs,
+    feature_group_name,
+    get_dataset_type,
+    get_delta_datestr_from_timestamp,
+    get_feature_group_url,
+    get_host_name,
+    get_hostname_replaced_url,
+    get_hudi_datestr_from_timestamp,
+    get_job_url,
+    get_timestamp_from_date_string,
+    is_interactive,
+    is_runtime_notebook,
+    run_with_loading_animation,
+    setup_pydoop,
+    strip_feature_store_suffix,
+    validate_embedding_feature_type,
+    validate_job_conf,
+    verify_attribute_key_names,
+]