[FSTORE-1389] get_secrets_api and create_project should be accessible as hopsworks module functions

Author: robzor92

auto_doc.py

@@ -22,6 +22,7 @@
 PAGES = {
     "api/login.md": {
         "login": ["hopsworks.login"],
+        "get_current_project": ["hopsworks.get_current_project"],
         "fs_api": ["hopsworks.project.Project.get_feature_store"],
         "mr_api": ["hopsworks.project.Project.get_model_registry"],
         "ms_api": ["hopsworks.project.Project.get_model_serving"],
@@ -36,9 +37,7 @@
         ),
     },
     "api/projects.md": {
-        "project_create": ["hopsworks.connection.Connection.create_project"],
-        "project_get": ["hopsworks.connection.Connection.get_project"],
-        "project_get_all": ["hopsworks.connection.Connection.get_projects"],
+        "project_create": ["hopsworks.create_project"],
         "project_properties": keras_autodoc.get_properties("hopsworks.project.Project"),
         "project_methods": keras_autodoc.get_methods(
             "hopsworks.project.Project", exclude=["from_response_json", "json"]
@@ -163,9 +162,10 @@
         ),
     },
     "api/secrets.md": {
-        "secret_api_handle": ["hopsworks.connection.Connection.get_secrets_api"],
+        "secret_api_handle": ["hopsworks.get_secrets_api"],
         "secret_create": ["hopsworks.core.secret_api.SecretsApi.create_secret"],
         "secret_get": ["hopsworks.core.secret_api.SecretsApi.get_secret"],
+        "secret_get_simplified": ["hopsworks.core.secret_api.SecretsApi.get"],
         "secret_get_all": ["hopsworks.core.secret_api.SecretsApi.get_secrets"],
         "secret_properties": keras_autodoc.get_properties("hopsworks.secret.Secret"),
         "secret_methods": keras_autodoc.get_methods(

docs/templates/api/login.md

@@ -2,6 +2,8 @@
 
 {{login}}
 
+{{get_current_project}}
+
 ## Feature Store API
 
 {{fs_api}}

docs/templates/api/projects.md

@@ -4,12 +4,6 @@
 
 {{project_create}}
 
-## Retrieval
-
-{{project_get}}
-
-{{project_get_all}}
-
 ## Properties
 
 {{project_properties}}

docs/templates/api/secrets.md

@@ -10,6 +10,8 @@
 
 ## Retrieval
 
+{{secret_get_simplified}}
+
 {{secret_get}}
 
 {{secret_get_all}}

python/hopsworks/__init__.py

@@ -23,9 +23,11 @@
 import warnings
 from pathlib import Path
 
-from hopsworks import client, constants, project, version
-from hopsworks.client.exceptions import ProjectException, RestAPIError
+from hopsworks.client.exceptions import RestAPIError, ProjectException
+from hopsworks import version, constants, client, project
 from hopsworks.connection import Connection
+from hopsworks.core import project_api, secret_api
+from hopsworks.decorators import NoHopsworksConnectionError
 
 
 # Needs to run before import of hsml and hsfs
@@ -42,7 +44,8 @@
 _hw_connection = Connection.connection
 
 _connected_project = None
-
+_secrets_api = None
+_project_api = None
 
 def hw_formatwarning(message, category, filename, lineno, line=None):
     return "{}: {}\n".format(category.__name__, message)
@@ -113,6 +116,7 @@ def login(
     if "REST_ENDPOINT" in os.environ:
         _hw_connection = _hw_connection()
         _connected_project = _hw_connection.get_project()
+        _initialize_module_apis()
         print("\nLogged in to project, explore it here " + _connected_project.get_url())
         return _connected_project
 
@@ -140,6 +144,8 @@ def login(
     elif host is None:  # Always do a fallback to Serverless Hopsworks if not defined
         host = constants.HOSTS.APP_HOST
 
+    is_app = (host == constants.HOSTS.APP_HOST)
+
     # If port same as default, get HOPSWORKS_HOST environment variable
     if port == 443 and "HOPSWORKS_PORT" in os.environ:
         port = os.environ["HOPSWORKS_PORT"]
@@ -166,23 +172,24 @@ def login(
                 "Could not find api key file on path: {}".format(api_key_file)
             )
     # If user connected to Serverless Hopsworks, and the cached .hw_api_key exists, then use it.
-    elif os.path.exists(api_key_path) and host == constants.HOSTS.APP_HOST:
+    elif os.path.exists(api_key_path) and is_app:
         try:
             _hw_connection = _hw_connection(
                 host=host, port=port, api_key_file=api_key_path
             )
-            _connected_project = _prompt_project(_hw_connection, project)
+            _connected_project = _prompt_project(_hw_connection, project, is_app)
             print(
                 "\nLogged in to project, explore it here "
                 + _connected_project.get_url()
             )
+            _initialize_module_apis()
             return _connected_project
         except RestAPIError:
             logout()
             # API Key may be invalid, have the user supply it again
             os.remove(api_key_path)
 
-    if api_key is None and host == constants.HOSTS.APP_HOST:
+    if api_key is None and is_app:
         print(
             "Copy your Api Key (first register/login): https://c.app.hopsworks.ai/account/api/generated"
         )
@@ -198,12 +205,17 @@ def login(
 
     try:
         _hw_connection = _hw_connection(host=host, port=port, api_key_value=api_key)
-        _connected_project = _prompt_project(_hw_connection, project)
+        _connected_project = _prompt_project(_hw_connection, project, is_app)
     except RestAPIError as e:
         logout()
         raise e
 
-    print("\nLogged in to project, explore it here " + _connected_project.get_url())
+    if _connected_project is None:
+        print("Could not find any project, use hopsworks.create_project('my_project') to create one")
+    else:
+        print("\nLogged in to project, explore it here " + _connected_project.get_url())
+
+    _initialize_module_apis()
     return _connected_project
 
 
@@ -245,11 +257,14 @@ def _get_cached_api_key_path():
     return api_key_path
 
 
-def _prompt_project(valid_connection, project):
+def _prompt_project(valid_connection, project, is_app):
     saas_projects = valid_connection.get_projects()
     if project is None:
         if len(saas_projects) == 0:
-            raise ProjectException("Could not find any project")
+            if is_app:
+                raise ProjectException("Could not find any project")
+            else:
+                return None
         elif len(saas_projects) == 1:
             return saas_projects[0]
         else:
@@ -258,7 +273,7 @@ def _prompt_project(valid_connection, project):
                 for index in range(len(saas_projects)):
                     print("\t (" + str(index + 1) + ") " + saas_projects[index].name)
                 while True:
-                    project_index = input("\nEnter project to access: ")
+                    project_index = input("\nEnter number corresponding to the project to use: ")
                     # Handle invalid input type
                     try:
                         project_index = int(project_index)
@@ -285,8 +300,98 @@ def _prompt_project(valid_connection, project):
 
 
 def logout():
+    """Cleans up and closes the connection for the hopsworks, hsfs and hsml libraries.
+    """
     global _hw_connection
-    if isinstance(_hw_connection, Connection):
+    global _project_api
+    global _secrets_api
+
+    if _is_connection_active():
         _hw_connection.close()
+
     client.stop()
+    _project_api = None
+    _secrets_api = None
     _hw_connection = Connection.connection
+
+def _is_connection_active():
+    global _hw_connection
+    return isinstance(_hw_connection, Connection)
+
+def get_current_project() -> project.Project:
+    """Get a reference to the current logged in project.
+
+    Example for creating a new project
+
+    ```python
+
+    import hopsworks
+
+    hopsworks.login()
+
+    project = hopsworks.get_current_project()
+
+    ```
+
+    # Returns
+        `Project`. A project handle object to perform operations on.
+    """
+    global _connected_project
+    if _connected_project is None:
+        raise ProjectException("No project is set for this session")
+    return _connected_project
+
+def _initialize_module_apis():
+    global _project_api
+    global _secrets_api
+    _project_api = project_api.ProjectApi()
+    _secrets_api = secret_api.SecretsApi()
+
+def create_project(
+        name: str, description: str = None, feature_store_topic: str = None
+):
+    """Create a new project.
+
+    Example for creating a new project
+
+    ```python
+
+    import hopsworks
+
+    hopsworks.login()
+
+    hopsworks.create_project("my_hopsworks_project", description="An example Hopsworks project")
+
+    ```
+    # Arguments
+        name: The name of the project.
+        description: optional description of the project
+        feature_store_topic: optional feature store topic name
+
+    # Returns
+        `Project`. A project handle object to perform operations on.
+    """
+    global _hw_connection
+    global _connected_project
+
+    if not _is_connection_active():
+        raise NoHopsworksConnectionError()
+
+    new_project = _hw_connection._project_api._create_project(name, description, feature_store_topic)
+    if _connected_project is None:
+        _connected_project = new_project
+        print("Setting {} as the current project, a reference can be retrieved by calling hopsworks.get_current_project()".format(_connected_project.name))
+        return _connected_project
+    else:
+        print("You are already using the project {}, to access the new project use hopsworks.login(..., project='{}')".format(_connected_project.name, new_project.name))
+
+def get_secrets_api():
+    """Get the secrets api.
+
+    # Returns
+        `SecretsApi`: The Secrets Api handle
+    """
+    global _secrets_api
+    if not _is_connection_active():
+        raise NoHopsworksConnectionError()
+    return _secrets_api

python/hopsworks/core/secret_api.py

@@ -14,9 +14,11 @@
 #   limitations under the License.
 #
 
-from hopsworks import client, secret
+from hopsworks import client, secret, util
 from hopsworks.core import project_api
 import json
+import getpass
+from hopsworks.client.exceptions import RestAPIError
 
 
 class SecretsApi:
@@ -42,11 +44,11 @@ def get_secrets(self):
             _client._send_request("GET", path_params)
         )
 
-    def get_secret(self, name: str, owner: str = None):
+    def get_secret(self, name: str, owner: str = None) -> secret.Secret:
         """Get a secret.
 
         # Arguments
-            name: Name of the project.
+            name: Name of the secret.
             owner: email of the owner for a secret shared with the current project.
         # Returns
             `Secret`: The Secret object
@@ -69,11 +71,34 @@ def get_secret(self, name: str, owner: str = None):
                 "shared",
             ]
 
-        return secret.Secret.from_response_json(
-            _client._send_request("GET", path_params, query_params=query_params)
-        )[0]
+        return secret.Secret.from_response_json(_client._send_request("GET", path_params, query_params=query_params))[0]
+
+    def get(self, name: str, owner: str = None) -> str:
+        """Get the secret's value.
+        If the secret does not exist, it prompts the user to create the secret if the application is running interactively
 
-    def create_secret(self, name: str, value: str, project: str = None):
+        # Arguments
+            name: Name of the secret.
+            owner: email of the owner for a secret shared with the current project.
+        # Returns
+            `str`: The secret value
+        # Raises
+            `RestAPIError`: If unable to get the secret
+        """
+        try:
+            return self.get_secret(name=name, owner=owner).value
+        except RestAPIError as e:
+            if (
+                    e.response.json().get("errorCode", "") == 160048
+                    and e.response.status_code == 404
+                    and util.is_interactive()
+            ):
+                secret_input = getpass.getpass(prompt="\nCould not find secret, enter value here to create it: ")
+                return self.create_secret(name, secret_input).value
+            else:
+                raise e
+
+    def create_secret(self, name: str, value: str, project: str = None) -> secret.Secret:
         """Create a new secret.
 
         ```python

python/hopsworks/project.py

@@ -109,6 +109,8 @@ def get_feature_store(self, name: str = None) -> feature_store.FeatureStore:
         Defaulting to the project name of default feature store. To get a
         shared feature store, the project name of the feature store is required.
 
+        # Arguments
+            name: Project name of the feature store.
         # Returns
             `hsfs.feature_store.FeatureStore`: The Feature Store API
         # Raises

python/hopsworks/secret.py

@@ -48,7 +48,7 @@ def __init__(
     @classmethod
     def from_response_json(cls, json_dict):
         json_decamelized = humps.decamelize(json_dict)
-        if len(json_decamelized["items"]) == 0:
+        if "items" not in json_decamelized or len(json_decamelized["items"]) == 0:
             return []
         return [cls(**secret) for secret in json_decamelized["items"]]
 

python/hopsworks/util.py

@@ -79,3 +79,7 @@ def get_hostname_replaced_url(sub_path: str):
     href = urljoin(client.get_instance()._base_url, sub_path)
     url_parsed = client.get_instance().replace_public_host(urlparse(href))
     return url_parsed.geturl()
+
+def is_interactive():
+    import __main__ as main
+    return not hasattr(main, '__file__')