Merge branch 'main' into aliases

Author: aversey

.github/workflows/python.yml

@@ -83,8 +83,14 @@ jobs:
         python-version: ["3.8", "3.9", "3.10"]
 
     steps:
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "adopt"
+
       - name: Set Timezone
-        run: sudo timedatectl set-timezone UTC
+        run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone
 
       - uses: actions/checkout@v4
       - name: Copy README
@@ -110,8 +116,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "adopt"
+
       - name: Set Timezone
-        run: sudo timedatectl set-timezone UTC
+        run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone
 
       - uses: actions/checkout@v4
       - name: Copy README
@@ -134,8 +146,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "adopt"
+
       - name: Set Timezone
-        run: sudo timedatectl set-timezone UTC
+        run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone
 
       - uses: actions/checkout@v4
       - name: Copy README
@@ -161,8 +179,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "adopt"
+
       - name: Set Timezone
-        run: sudo timedatectl set-timezone Europe/Amsterdam
+        run: sudo timedatectl set-timezone Europe/Amsterdam && echo Europe/Amsterdam | sudo tee /etc/timezone
 
       - uses: actions/checkout@v4
       - name: Copy README

CONTRIBUTING.md

@@ -36,7 +36,7 @@
 
 We follow a few best practices for writing the Python documentation:
 
-1. Use the google docstring style:
+1. Use the Google docstring style:
 
    ```python
    """[One Line Summary]
@@ -72,16 +72,17 @@ We use `mkdocs` together with `mike` ([for versioning](https://github.com/jimpor
 **Background about `mike`:**
 `mike` builds the documentation and commits it as a new directory to the gh-pages branch. Each directory corresponds to one version of the documentation. Additionally, `mike` maintains a json in the root of gh-pages with the mappings of versions/aliases for each of the directories available. With aliases you can define extra names like `dev` or `latest`, to indicate stable and unstable releases.
 
-1. Install Hopsworks with `dev-docs` extras:
+1. Install Hopsworks with `requirements-docs.txt`:
 
    ```bash
-   pip install -e ".[dev-docs]"
+   pip install -r requirements-docs.txt
+   pip install -e "python[dev]"
    ```
 
 2. To build the docs, first run the auto doc script:
 
    ```bash
-   python auto_doc.py
+   python python/auto_doc.py
    ```
 
 ##### Option 1: Build only current version of docs

README.md

@@ -164,6 +164,23 @@ data = { "instances": [ model.input_example ] }
 predictions = deployment.predict(data)
 ```
 
+## Usage
+
+Usage data is collected for improving quality of the library.
+It is turned on by default if the backend is [Hopsworks Serverless](https://c.app.hopsworks.ai).
+To turn it off, use one of the following ways:
+```python
+# use environment variable
+import os
+os.environ["ENABLE_HOPSWORKS_USAGE"] = "false"
+
+# use `disable_usage_logging`
+import hopsworks
+hopsworks.disable_usage_logging()
+```
+
+The corresponding source code is in `python/hopsworks_common/usage.py`.
+
 ## Tutorials
 
 Need more inspiration or want to learn more about the Hopsworks platform? Check out our [tutorials](https://github.com/logicalclocks/hopsworks-tutorials).
@@ -176,17 +193,12 @@ Documentation is available at [Hopsworks Documentation](https://docs.hopsworks.a
 
 For general questions about the usage of Hopsworks and the Feature Store please open a topic on [Hopsworks Community](https://community.hopsworks.ai/).
 
-Please report any issue using [Github issue tracking](https://github.com/logicalclocks/hopsworks-api/issues).
-
-### Related to Feautre Store API
-
-Please attach the client environment from the output below to your issue, if it is related to Feature Store API:
+Please report any issue using [Github issue tracking](https://github.com/logicalclocks/hopsworks-api/issues) and attach the client environment from the output below to your issue:
 
 ```python
 import hopsworks
-import hsfs
-hopsworks.login().get_feature_store()
-print(hsfs.get_env())
+hopsworks.login()
+print(hopsworks.get_sdk_info())
 ```
 
 ## Contributing

docs/templates/api/connection.md

@@ -1,13 +1,11 @@
-# Connection API
+# Connection
 
-## Creation
-
-{{connection_create}}
+{{connection}}
 
 ## Properties
 
 {{connection_properties}}
 
 ## Methods
 
-{{connection_methods}}
+{{connection_methods}}

docs/templates/api/connection_api.md

@@ -1,6 +1,6 @@
 # Jobs API
 
-## Handle 
+## Handle
 
 {{job_api_handle}}
 

docs/templates/api/job.md

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>4.0.0-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

docs/templates/api/jobs.md

@@ -25,6 +25,7 @@
 import com.logicalclocks.hsfs.OnlineConfig;
 import com.logicalclocks.hsfs.StatisticsConfig;
 import com.logicalclocks.hsfs.Storage;
+import com.logicalclocks.hsfs.StorageConnector;
 import com.logicalclocks.hsfs.beam.engine.FeatureGroupEngine;
 import com.logicalclocks.hsfs.beam.engine.BeamProducer;
 import com.logicalclocks.hsfs.constructor.QueryBase;
@@ -48,7 +49,7 @@ public class StreamFeatureGroup extends FeatureGroupBase<PCollection<Object>> {
   public StreamFeatureGroup(FeatureStore featureStore, @NonNull String name, Integer version, String description,
       List<String> primaryKeys, List<String> partitionKeys, String hudiPrecombineKey,
       boolean onlineEnabled, List<Feature> features, StatisticsConfig statisticsConfig, String onlineTopicName,
-      String eventTime, OnlineConfig onlineConfig) {
+      String eventTime, OnlineConfig onlineConfig, StorageConnector storageConnector, String path) {
     this();
     this.featureStore = featureStore;
     this.name = name;
@@ -65,6 +66,8 @@ public StreamFeatureGroup(FeatureStore featureStore, @NonNull String name, Integ
     this.onlineTopicName = onlineTopicName;
     this.eventTime = eventTime;
     this.onlineConfig = onlineConfig;
+    this.storageConnector = storageConnector;
+    this.path = path;
   }
 
   public StreamFeatureGroup() {

docs/templates/connection_api.md

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>4.0.0-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/beam/pom.xml

@@ -26,6 +26,7 @@
 import com.logicalclocks.hsfs.OnlineConfig;
 import com.logicalclocks.hsfs.StatisticsConfig;
 import com.logicalclocks.hsfs.Storage;
+import com.logicalclocks.hsfs.StorageConnector;
 import com.logicalclocks.hsfs.constructor.QueryBase;
 
 import com.logicalclocks.hsfs.metadata.Statistics;
@@ -54,7 +55,7 @@ public StreamFeatureGroup(FeatureStore featureStore, @NonNull String name, Integ
       List<String> primaryKeys, List<String> partitionKeys, String hudiPrecombineKey,
       boolean onlineEnabled, List<Feature> features, StatisticsConfig statisticsConfig,
       String onlineTopicName, String topicName, String notificationTopicName, String eventTime,
-      OnlineConfig onlineConfig) {
+      OnlineConfig onlineConfig, StorageConnector storageConnector, String path) {
     this();
     this.featureStore = featureStore;
     this.name = name;
@@ -73,6 +74,8 @@ public StreamFeatureGroup(FeatureStore featureStore, @NonNull String name, Integ
     this.notificationTopicName = notificationTopicName;
     this.eventTime = eventTime;
     this.onlineConfig = onlineConfig;
+    this.storageConnector = storageConnector;
+    this.path = path;
   }
 
   public StreamFeatureGroup() {

java/beam/src/main/java/com/logicalclocks/hsfs/beam/StreamFeatureGroup.java

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>4.0.0-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/flink/pom.xml

@@ -132,6 +132,14 @@ public abstract class FeatureGroupBase<T> {
   @Setter
   protected OnlineConfig onlineConfig;
 
+  @Getter
+  @Setter
+  protected StorageConnector storageConnector;
+
+  @Getter
+  @Setter
+  protected String path;
+
   @JsonIgnore
   // These are only used in the client. In the server they are aggregated in the `features` field
   protected List<String> partitionKeys;

java/flink/src/main/java/com/logicalclocks/hsfs/flink/StreamFeatureGroup.java

@@ -124,6 +124,9 @@ public static class S3Connector extends StorageConnector {
     @Getter @Setter
     protected String bucket;
 
+    @Getter @Setter
+    protected String region;
+
     @Getter @Setter
     protected String sessionToken;
 

java/hsfs/pom.xml

@@ -128,7 +128,7 @@ public static Long getTimeStampFromDateString(String inputDate) throws FeatureSt
     }
 
     SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatPattern);
-    Long commitTimeStamp = dateFormat.parse(tempDate).getTime();;
+    Long commitTimeStamp = dateFormat.parse(tempDate).getTime();
 
     return commitTimeStamp;
   }

java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java

@@ -22,12 +22,14 @@
 import org.apache.http.HttpRequest;
 import org.apache.http.client.ResponseHandler;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.Charset;
 
 public interface HopsworksHttpClient {
-  static final Logger LOGGER = null;
+  Logger LOGGER = LoggerFactory.getLogger(HopsworksHttpClient.class);
 
   <T> T handleRequest(HttpRequest request, ResponseHandler<T> responseHandler)
       throws IOException, FeatureStoreException;
@@ -46,7 +48,7 @@ <T> T handleRequest(HttpRequest request, ResponseHandler<T> responseHandler)
 
   static String readCertKey(String materialPwd) {
     try {
-      return FileUtils.readFileToString(new File(materialPwd));
+      return FileUtils.readFileToString(new File(materialPwd), Charset.defaultCharset());
     } catch (IOException ex) {
       LOGGER.warn("Failed to get cert password.", ex);
     }

java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnector.java

@@ -46,4 +46,12 @@ public void testStringEntitySerialization() throws IOException {
     Assertions.assertEquals("{\"email\":\"test@test.com\",\"firstName\":\"test\",\"lastName\":\"de la Rúa Martínez\"}",
         json);
   }
+
+  // FSTORE-1562: readCertKey throws NullPointerException if
+  @Test
+  public void testReadCertKey_failure() {
+    String key = HopsworksHttpClient.readCertKey("/this/path/does/not/exists");
+    Assertions.assertNull(key);
+  }
+
 }

java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java

@@ -7,7 +7,7 @@
     <groupId>com.logicalclocks</groupId>
     <artifactId>hsfs-parent</artifactId>
     <packaging>pom</packaging>
-    <version>4.0.0-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
     <modules>
         <module>hsfs</module>
         <module>spark</module>

java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>hsfs-parent</artifactId>
         <groupId>com.logicalclocks</groupId>
-        <version>4.0.0-SNAPSHOT</version>
+        <version>4.1.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java

@@ -57,10 +57,6 @@
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class ExternalFeatureGroup extends FeatureGroupBase<Dataset<Row>> {
 
-  @Getter
-  @Setter
-  private StorageConnector storageConnector;
-
   @Getter
   @Setter
   private String query;
@@ -69,10 +65,6 @@ public class ExternalFeatureGroup extends FeatureGroupBase<Dataset<Row>> {
   @Setter
   private ExternalDataFormat dataFormat;
 
-  @Getter
-  @Setter
-  private String path;
-
   @Getter
   @Setter
   private List<OnDemandOptions> options;