Merge branch 'main' into FSTORE-1537

Author: bubriks

CONTRIBUTING.md

@@ -36,7 +36,7 @@
 
 We follow a few best practices for writing the Python documentation:
 
-1. Use the google docstring style:
+1. Use the Google docstring style:
 
    ```python
    """[One Line Summary]
@@ -72,16 +72,17 @@ We use `mkdocs` together with `mike` ([for versioning](https://github.com/jimpor
 **Background about `mike`:**
 `mike` builds the documentation and commits it as a new directory to the gh-pages branch. Each directory corresponds to one version of the documentation. Additionally, `mike` maintains a json in the root of gh-pages with the mappings of versions/aliases for each of the directories available. With aliases you can define extra names like `dev` or `latest`, to indicate stable and unstable releases.
 
-1. Install Hopsworks with `dev-docs` extras:
+1. Install Hopsworks with `requirements-docs.txt`:
 
    ```bash
-   pip install -e ".[dev-docs]"
+   pip install -r requirements-docs.txt
+   pip install -e "python[dev]"
    ```
 
 2. To build the docs, first run the auto doc script:
 
    ```bash
-   python auto_doc.py
+   python python/auto_doc.py
    ```
 
 ##### Option 1: Build only current version of docs

docs/templates/api/connection.md

@@ -1,13 +1,11 @@
-# Connection API
+# Connection
 
-## Creation
-
-{{connection_create}}
+{{connection}}
 
 ## Properties
 
 {{connection_properties}}
 
 ## Methods
 
-{{connection_methods}}
+{{connection_methods}}

docs/templates/api/connection_api.md

@@ -1,6 +1,6 @@
 # Jobs API
 
-## Handle 
+## Handle
 
 {{job_api_handle}}
 

docs/templates/api/job.md

@@ -16,8 +16,8 @@ nav:
   - Guides: https://docs.hopsworks.ai/
   - Setup and Installation: https://docs.hopsworks.ai/
   - Administration: https://docs.hopsworks.ai/
-  - API<div class="dropdown"><button class="dropbtn"> API </button> <div id="myDropdown" class="dropdown-content"> <a id="hopsworks_api_link" href="https://docs.hopsworks.ai/hopsworks-api/latest">Hopsworks API</a> <a id="hsfs_api_link" href="https://docs.hopsworks.ai/feature-store-api/latest">Feature Store API</a> <a id="hsfs_javadoc_link" href="https://docs.hopsworks.ai/feature-store-api/latest/javadoc">Feature Store JavaDoc</a> <a id="hsml_api_link" href="https://docs.hopsworks.ai/machine-learning-api/latest">MLOps API</a> </div></div>:
-    - API Reference:
+  - API<div class="dropdown"><button class="dropbtn"> API </button> <div id="myDropdown" class="dropdown-content"> <a id="hopsworks_api_link" href="https://docs.hopsworks.ai/hopsworks-api/latest">Python API</a> <a id="hsfs_javadoc_link" href="https://docs.hopsworks.ai/hopsworks-api/latest/javadoc">Feature Store JavaDoc</a> </div></div>:
+    - Platform API:
       - Login: generated/api/login.md
       - Connection: generated/api/connection.md
       - Projects: generated/api/projects.md
@@ -33,7 +33,7 @@ nav:
       - KafkaSchema: generated/api/kafka_schema.md
       - Secrets: generated/api/secrets.md
       - OpenSearch: generated/api/opensearch.md
-      - Connection (HSFS): generated/api/connection_api.md
+    - Feature Store API:
       - ExpectationSuite: generated/api/expectation_suite_api.md
       - FeatureStore: generated/api/feature_store_api.md
       - FeatureGroup: generated/api/feature_group_api.md
@@ -48,11 +48,10 @@ nav:
         - UDF: generated/api/udf.md
         - HopsworksUDF: generated/api/hopsworks_udf.md
         - TransformationFunction: generated/api/transformation_functions_api.md
-        - Transformation Statistics: 
+        - Transformation Statistics:
           - TransformationStatistics: generated/api/transformation_statistics.md
           - FeatureTransformationStatistics: generated/api/feature_transformation_statistics.md
       - ValidationReport: generated/api/validation_report_api.md
-      - Job: generated/api/job.md
       - Provenance Links: generated/api/links.md
       - Statistics:
           - Statistics: generated/api/statistics_api.md
@@ -66,7 +65,7 @@ nav:
           - EmbeddingIndex: generated/api/embedding_index_api.md
           - EmbeddingFeature: generated/api/embedding_feature_api.md
           - SimilarityFunctionType: generated/api/similarity_function_type_api.md
-      - Connection (HSML): generated/connection_api.md
+    - Machine Learning API:
       - Model Registry:
         - Model Registry: generated/model-registry/model_registry_api.md
         - Model: generated/model-registry/model_api.md
@@ -82,7 +81,6 @@ nav:
         - Inference Batcher: generated/model-serving/inference_batcher_api.md
         - Resources: generated/model-serving/resources_api.md
     # Added to allow navigation using the side drawer
-    - Hopsworks API: https://docs.hopsworks.ai/hopsworks-api/latest/
     - Feature Store JavaDoc: https://docs.hopsworks.ai/feature-store-javadoc/latest/
     - Contributing: CONTRIBUTING.md
   - Community ↗: https://community.hopsworks.ai/

docs/templates/api/jobs.md

@@ -40,7 +40,7 @@
         "udf": ["hopsworks.udf"],
     },
     "api/connection.md": {
-        "connection_create": ["hopsworks.connection.Connection.connection"],
+        "connection": ["hopsworks.connection.Connection.connection"],
         "connection_properties": keras_autodoc.get_properties(
             "hopsworks.connection.Connection"
         ),
@@ -61,7 +61,10 @@
         "job_get": ["hopsworks.core.job_api.JobApi.get_job"],
         "job_get_all": ["hopsworks.core.job_api.JobApi.get_jobs"],
         "job_properties": keras_autodoc.get_properties("hopsworks.job.Job"),
-        "job_config": ["hopsworks.core.job_api.JobApi.get_configuration"],
+        "job_config": [
+            "hopsworks.core.job_api.JobApi.get_configuration",
+            "hopsworks_common.core.job_configuration.JobConfiguration",
+        ],
         "job_methods": keras_autodoc.get_methods(
             "hopsworks.job.Job", exclude=["from_response_json", "json"]
         ),
@@ -190,13 +193,6 @@
             "hopsworks.core.opensearch_api.OpenSearchApi"
         ),
     },
-    "api/connection_api.md": {
-        "connection": ["hsfs.connection.Connection"],
-        "connection_properties": keras_autodoc.get_properties(
-            "hsfs.connection.Connection"
-        ),
-        "connection_methods": keras_autodoc.get_methods("hsfs.connection.Connection"),
-    },
     "api/spine_group_api.md": {
         "fg": ["hsfs.feature_group.SpineGroup"],
         "fg_create": ["hsfs.feature_store.FeatureStore.get_or_create_spine_group"],
@@ -413,14 +409,6 @@
             "hsfs.validation_report.ValidationReport"
         ),
     },
-    "api/job.md": {
-        "job_configuration": ["hsfs.core.job_configuration.JobConfiguration"],
-        "job": ["hsfs.core.job.Job"],
-        "job_methods": [
-            "hsfs.core.job.Job.get_state",
-            "hsfs.core.job.Job.get_final_state",
-        ],
-    },
     "api/query_api.md": {
         "query_methods": keras_autodoc.get_methods(
             "hsfs.constructor.query.Query",
@@ -550,13 +538,6 @@
         "similarity_function_type": ["hsfs.embedding.SimilarityFunctionType"],
     },
     # Model registry
-    "connection_api.md": {
-        "connection": ["hsml.connection.Connection"],
-        "connection_properties": keras_autodoc.get_properties(
-            "hsml.connection.Connection", exclude=["trust_store_path"]
-        ),
-        "connection_methods": keras_autodoc.get_methods("hsml.connection.Connection"),
-    },
     "model-registry/model_registry_api.md": {
         "mr_get": ["hsml.connection.Connection.get_model_registry"],
         "mr_modules": keras_autodoc.get_properties(

docs/templates/connection_api.md

@@ -21,6 +21,12 @@
 from hopsworks_common.client import auth, base
 
 
+try:
+    import jks
+except ImportError:
+    pass
+
+
 class Client(base.Client):
     HOPSWORKS_HOSTNAME_VERIFICATION = "HOPSWORKS_HOSTNAME_VERIFICATION"
     DOMAIN_CA_TRUSTSTORE_PEM = "DOMAIN_CA_TRUSTSTORE_PEM"
@@ -50,7 +56,7 @@ def __init__(self, hostname_verification):
         self._hostname_verification = os.environ.get(
             self.HOPSWORKS_HOSTNAME_VERIFICATION, "{}".format(hostname_verification)
         ).lower() in ("true", "1", "y", "yes")
-        self._hopsworks_ca_trust_store_path = self._get_ca_chain_path()
+        self._hopsworks_ca_trust_store_path = self._materialize_ca_chain()
 
         self._project_id = os.environ[self.PROJECT_ID]
         self._project_name = self._project_name()
@@ -67,10 +73,23 @@ def __init__(self, hostname_verification):
 
         credentials = self._get_credentials(self._project_id)
 
-        self._write_pem_file(credentials["caChain"], self._get_ca_chain_path())
         self._write_pem_file(credentials["clientCert"], self._get_client_cert_path())
         self._write_pem_file(credentials["clientKey"], self._get_client_key_path())
 
+    def _materialize_ca_chain(self):
+        """Convert truststore from jks to pem and return the location"""
+        ca_chain_path = Path(self._get_ca_chain_path())
+        if not ca_chain_path.exists():
+            keystore_pw = self._cert_key
+            ks = jks.KeyStore.load(
+                self._get_jks_key_store_path(), keystore_pw, try_decrypt_keys=True
+            )
+            ts = jks.KeyStore.load(
+                self._get_jks_trust_store_path(), keystore_pw, try_decrypt_keys=True
+            )
+            self._write_ca_chain(ks, ts, ca_chain_path)
+        return str(ca_chain_path)
+
     def _get_hopsworks_rest_endpoint(self):
         """Get the hopsworks REST endpoint for making requests to the REST API."""
         return os.environ[self.REST_ENDPOINT]

mkdocs.yml

@@ -38,7 +38,7 @@
 
 HOPSWORKS_PORT_DEFAULT = 443
 HOSTNAME_VERIFICATION_DEFAULT = os.environ.get(
-    "HOPSWORKS_HOSTNAME_VERIFICATION", "True"
+    "HOPSWORKS_HOSTNAME_VERIFICATION", "False"
 ).lower() in ("true", "1", "y", "yes")
 # alias for backwards compatibility:
 HOPSWORKS_HOSTNAME_VERIFICATION_DEFAULT = HOSTNAME_VERIFICATION_DEFAULT

python/auto_doc.py

@@ -25,25 +25,35 @@
 HAS_CONFLUENT_KAFKA: bool = importlib.util.find_spec("confluent_kafka") is not None
 confluent_kafka_not_installed_message = (
     "Confluent Kafka package not found. "
-    "If you want to use Kafka with Hopsworks you can install the corresponding extras "
-    """`pip install hopsworks[python]` or `pip install "hopsworks[python]"` if using zsh. """
-    "You can also install confluent-kafka directly in your environment e.g `pip install confluent-kafka`. "
+    "If you want to use Kafka with Hopsworks you can install the corresponding extras via "
+    '`pip install "hopsworks[python]"`. '
+    "You can also install confluent-kafka directly in your environment with `pip install confluent-kafka`. "
     "You will need to restart your kernel if applicable."
 )
+
 # Data Validation / Great Expectations
 HAS_GREAT_EXPECTATIONS: bool = (
     importlib.util.find_spec("great_expectations") is not None
 )
 great_expectations_not_installed_message = (
     "Great Expectations package not found. "
-    "If you want to use data validation with Hopsworks you can install the corresponding extras "
-    """`pip install hopsworks[great_expectations]` or `pip install "hopsworks[great_expectations]"` if using zsh. """
-    "You can also install great-expectations directly in your environment e.g `pip install great-expectations`. "
+    "If you want to use data validation with Hopsworks you can install the corresponding extras via "
+    '`pip install "hopsworks[great_expectations]"`. '
+    "You can also install great-expectations directly in your environment with `pip install great-expectations`. "
     "You will need to restart your kernel if applicable."
 )
 initialise_expectation_suite_for_single_expectation_api_message = "Initialize Expectation Suite by attaching to a Feature Group to enable single expectation API"
 
-HAS_ARROW: bool = importlib.util.find_spec("pyarrow") is not None
+# Pyarrow
+HAS_PYARROW: bool = importlib.util.find_spec("pyarrow") is not None
+pyarrow_not_installed_message = (
+    "Pyarrow package not found. "
+    "If you want to use Apache Arrow with Hopsworks you can install the corresponding extras via "
+    '`pip install "hopsworks[python]"`. '
+    "You can also install pyarrow directly in your environment with `pip install pyarrow`. "
+    "You will need to restart your kernel if applicable."
+)
+
 HAS_PANDAS: bool = importlib.util.find_spec("pandas") is not None
 HAS_NUMPY: bool = importlib.util.find_spec("numpy") is not None
 HAS_POLARS: bool = importlib.util.find_spec("polars") is not None

python/hopsworks_common/client/hopsworks.py

@@ -23,9 +23,9 @@
 
 import pytz
 from hopsworks_common.core.constants import (
-    HAS_ARROW,
     HAS_PANDAS,
     HAS_POLARS,
+    HAS_PYARROW,
 )
 from hopsworks_common.decorators import uses_polars
 
@@ -35,7 +35,7 @@
     import pandas as pd
     import polars as pl
 
-if HAS_ARROW:
+if HAS_PYARROW:
     import pyarrow as pa
 
     # Decimal types are currently not supported

python/hopsworks_common/connection.py

@@ -51,6 +51,7 @@ def __init__(
         services=None,
         datasets=None,
         creation_status=None,
+        project_namespace=None,
         **kwargs,
     ):
         self._id = project_id
@@ -67,6 +68,7 @@ def __init__(
         self._git_api = git_api.GitApi()
         self._dataset_api = dataset_api.DatasetApi()
         self._environment_api = environment_api.EnvironmentApi()
+        self._project_namespace = project_namespace
 
     @classmethod
     def from_response_json(cls, json_dict):
@@ -101,6 +103,11 @@ def created(self):
         """Timestamp when the project was created"""
         return self._created
 
+    @property
+    def project_namespace(self):
+        """Kubernetes namespace used by project"""
+        return self._project_namespace
+
     def get_feature_store(
         self, name: Optional[str] = None, engine: Optional[str] = None
     ):  # -> hsfs.feature_store.FeatureStore

python/hopsworks_common/core/constants.py

@@ -29,7 +29,7 @@
 from hsfs.constructor import join
 from hsfs.constructor.filter import Filter, Logic
 from hsfs.constructor.fs_query import FsQuery
-from hsfs.core import arrow_flight_client, query_constructor_api, storage_connector_api
+from hsfs.core import query_constructor_api, storage_connector_api
 from hsfs.decorators import typechecked
 from hsfs.feature import Feature
 
@@ -101,6 +101,8 @@ def _prep_read(
             online_conn = None
 
             if engine.get_instance().is_flyingduck_query_supported(self, read_options):
+                from hsfs.core import arrow_flight_client
+
                 sql_query = self._to_string(fs_query, online, asof=True)
                 sql_query = arrow_flight_client.get_instance().create_query_object(
                     self, sql_query, fs_query.on_demand_fg_aliases

python/hopsworks_common/core/type_systems.py

@@ -23,6 +23,12 @@
 from functools import wraps
 from typing import Any, Dict, Optional, Union
 
+from hopsworks_common.core.constants import HAS_PYARROW, pyarrow_not_installed_message
+
+
+if not HAS_PYARROW:
+    raise ModuleNotFoundError(pyarrow_not_installed_message)
+
 import pyarrow
 import pyarrow._flight
 import pyarrow.flight