Merge branch 'main' into FSTORE-1564-new

Author: bubriks

python/hsfs/engine/python.py

@@ -1473,9 +1473,9 @@ def _start_offline_materialization(offline_write_options: Dict[str, Any]) -> boo
     def _convert_feature_log_to_df(feature_log, cols) -> pd.DataFrame:
         if feature_log is None and cols:
             return pd.DataFrame(columns=cols)
-        if not (isinstance(feature_log, (list, pd.DataFrame, pl.DataFrame))) or (
+        if not (isinstance(feature_log, (list, pd.DataFrame, pl.DataFrame)) or (
             HAS_NUMPY and isinstance(feature_log, np.ndarray)
-        ):
+        )):
             raise ValueError(f"Type '{type(feature_log)}' not accepted")
         if isinstance(feature_log, list) or (
             HAS_NUMPY and isinstance(feature_log, np.ndarray)

python/hsfs/storage_connector.py

@@ -373,12 +373,16 @@ def prepare_spark(self, path: Optional[str] = None) -> Optional[str]:
 
     def connector_options(self) -> Dict[str, Any]:
         """Return options to be passed to an external S3 connector library"""
-        return {
+        self.refetch()
+        options = {
             "access_key": self.access_key,
             "secret_key": self.secret_key,
             "session_token": self.session_token,
             "region": self.region,
         }
+        if self.arguments.get("fs.s3a.endpoint"):
+            options["endpoint"] = self.arguments.get("fs.s3a.endpoint")
+        return options
 
     def read(
         self,