[FSTORE-1439][APPEND] Apply ruff formatting and fixes to the merged repo (#232)

* Ruff fix

* Ruff format

* Ruff fix

* Revert making hsfs/client/base proper abstract class

Author: aversey

python/hopsworks/client/__init__.py

@@ -16,6 +16,7 @@
 
 from hopsworks.client import external, hopsworks
 
+
 _client = None
 _python_version = None
 

python/hopsworks/client/base.py

@@ -15,13 +15,12 @@
 #
 
 import os
-import furl
 from abc import ABC, abstractmethod
 
+import furl
 import requests
 import urllib3
-
-from hopsworks.client import exceptions, auth
+from hopsworks.client import auth, exceptions
 from hopsworks.decorators import connected
 
 

python/hopsworks/client/hopsworks.py

@@ -20,8 +20,8 @@
 from pathlib import Path
 
 import requests
+from hopsworks.client import auth, base
 
-from hopsworks.client import base, auth
 
 try:
     import jks
@@ -134,7 +134,7 @@ def _convert_jks_to_pem(self, jks_path, keystore_pw):
         ca_certs = ""
 
         # Convert CA Certificates into PEM format and append to string
-        for alias, c in ks.certs.items():
+        for _alias, c in ks.certs.items():
             ca_certs = ca_certs + self._bytes_to_pem_str(c.cert, "CERTIFICATE")
         return ca_certs
 

python/hopsworks/connection.py

@@ -16,14 +16,14 @@
 
 import os
 import re
-import warnings
 import sys
+import warnings
 
-from requests.exceptions import ConnectionError
-
-from hopsworks.decorators import connected, not_connected
 from hopsworks import client, version
 from hopsworks.core import project_api, secret_api, variable_api
+from hopsworks.decorators import connected, not_connected
+from requests.exceptions import ConnectionError
+
 
 HOPSWORKS_PORT_DEFAULT = 443
 HOSTNAME_VERIFICATION_DEFAULT = True
@@ -210,7 +210,8 @@ def _check_compatibility(self):
             warnings.warn(
                 "The installed hopsworks client version {0} may not be compatible with the connected Hopsworks backend version {1}. \nTo ensure compatibility please install the latest bug fix release matching the minor version of your backend ({2}) by running 'pip install hopsworks=={2}.*'".format(
                     client_version, backend_version, major_minor_backend
-                )
+                ),
+                stacklevel=1,
             )
             sys.stderr.flush()
 

python/hopsworks/core/dataset_api.py

@@ -14,18 +14,17 @@
 #   limitations under the License.
 #
 
+import copy
+import logging
 import math
 import os
-import time
-from tqdm.auto import tqdm
 import shutil
-import logging
-import copy
+import time
+from concurrent.futures import ThreadPoolExecutor, wait
 
 from hopsworks import client
-from hopsworks.client.exceptions import RestAPIError
-from hopsworks.client.exceptions import DatasetException
-from concurrent.futures import ThreadPoolExecutor, wait
+from hopsworks.client.exceptions import DatasetException, RestAPIError
+from tqdm.auto import tqdm
 
 
 class Chunk:

python/hopsworks/core/environment_api.py

@@ -32,7 +32,13 @@ def __init__(
 
         self._environment_engine = environment_engine.EnvironmentEngine(project_id)
 
-    def create_environment(self, name: str, description: Optional[str] = None, base_environment_name: Optional[str] = "python-feature-pipeline", await_creation: Optional[bool] = True) -> environment.Environment:
+    def create_environment(
+        self,
+        name: str,
+        description: Optional[str] = None,
+        base_environment_name: Optional[str] = "python-feature-pipeline",
+        await_creation: Optional[bool] = True,
+    ) -> environment.Environment:
         """Create Python environment for the project
 
         ```python
@@ -66,13 +72,14 @@ def create_environment(self, name: str, description: Optional[str] = None, base_
             name,
         ]
         headers = {"content-type": "application/json"}
-        data = {"name": name,
-                "baseImage": {
-                    "name": base_environment_name,
-                    "description": description
-                }}
+        data = {
+            "name": name,
+            "baseImage": {"name": base_environment_name, "description": description},
+        }
         env = environment.Environment.from_response_json(
-            _client._send_request("POST", path_params, headers=headers, data=json.dumps(data)),
+            _client._send_request(
+                "POST", path_params, headers=headers, data=json.dumps(data)
+            ),
             self._project_id,
             self._project_name,
         )
@@ -148,4 +155,4 @@ def _delete(self, name):
             name,
         ]
         headers = {"content-type": "application/json"}
-        _client._send_request("DELETE", path_params, headers=headers),
+        (_client._send_request("DELETE", path_params, headers=headers),)

python/hopsworks/core/flink_cluster_api.py

@@ -14,9 +14,11 @@
 #   limitations under the License.
 #
 
-import os
 import json
-from hopsworks import client, flink_cluster, util, job
+import os
+
+from hopsworks import client, flink_cluster, job, util
+from hopsworks.client.exceptions import RestAPIError
 from hopsworks.core import job_api
 
 
@@ -69,7 +71,9 @@ def setup_cluster(self, name: str, config=None):
             # If the job already exists, retrieve it
             _flink_cluster = self.get_cluster(name)
             if _flink_cluster._job.job_type != "FLINK":
-                raise "This is not a Flink cluster. Please use different name to create new Flink cluster"
+                raise RestAPIError(
+                    "This is not a Flink cluster. Please use different name to create new Flink cluster"
+                )
             return _flink_cluster
         else:
             # If the job doesn't exists, create a new job

python/hopsworks/core/git_api.py

@@ -14,23 +14,23 @@
 #   limitations under the License.
 #
 
+import json
+import logging
+from typing import List, Union
+
 from hopsworks import (
     client,
-    git_repo,
-    git_op_execution,
-    util,
     git_commit,
     git_file_status,
+    git_op_execution,
+    git_repo,
+    util,
 )
 from hopsworks.client.exceptions import GitException
-from hopsworks.engine import git_engine
 from hopsworks.core import git_provider_api
-from typing import List, Union
+from hopsworks.engine import git_engine
 from hopsworks.git_file_status import GitFileStatus
 
-import json
-import logging
-
 
 class GitApi:
     def __init__(
@@ -347,7 +347,7 @@ def _status(self, repo_id):
 
         status_dict = json.loads(git_op.command_result_message)
         file_status = None
-        if status_dict is not None and type(status_dict["status"]) is list:
+        if status_dict is not None and isinstance(status_dict["status"], list):
             file_status = []
             for status in status_dict["status"]:
                 file_status.append(

python/hopsworks/core/git_provider_api.py

@@ -14,11 +14,11 @@
 #   limitations under the License.
 #
 
+import json
+
 from hopsworks import client, git_provider
-from hopsworks.engine import git_engine
 from hopsworks.client.exceptions import GitException
-
-import json
+from hopsworks.engine import git_engine
 
 
 class GitProviderApi:

python/hopsworks/core/job_api.py

@@ -16,7 +16,7 @@
 
 import json
 
-from hopsworks import client, job, util, job_schedule
+from hopsworks import client, job, job_schedule, util
 from hopsworks.client.exceptions import RestAPIError
 
 

python/hopsworks/core/kafka_api.py

@@ -14,10 +14,11 @@
 #   limitations under the License.
 #
 
-from hopsworks import client, kafka_topic, kafka_schema, constants
-from hopsworks.client.exceptions import KafkaException
 import json
 import socket
+
+from hopsworks import client, constants, kafka_schema, kafka_topic
+from hopsworks.client.exceptions import KafkaException
 from hopsworks.client.external import Client
 
 
@@ -366,7 +367,7 @@ def get_default_config(self):
             constants.KAFKA_SSL_CONFIG.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG: "none",
         }
         _client = client.get_instance()
-        if type(_client) == Client:
+        if type(_client) is Client:
             config[constants.KAFKA_PRODUCER_CONFIG.BOOTSTRAP_SERVERS_CONFIG] = ",".join(
                 [
                     endpoint.replace("EXTERNAL://", "")

python/hopsworks/core/secret_api.py

@@ -72,7 +72,9 @@ def get_secret(self, name: str, owner: str = None) -> secret.Secret:
                 "shared",
             ]
 
-        return secret.Secret.from_response_json(_client._send_request("GET", path_params, query_params=query_params))[0]
+        return secret.Secret.from_response_json(
+            _client._send_request("GET", path_params, query_params=query_params)
+        )[0]
 
     def get(self, name: str, owner: str = None) -> str:
         """Get the secret's value.
@@ -90,16 +92,20 @@ def get(self, name: str, owner: str = None) -> str:
             return self.get_secret(name=name, owner=owner).value
         except RestAPIError as e:
             if (
-                    e.response.json().get("errorCode", "") == 160048
-                    and e.response.status_code == 404
-                    and util.is_interactive()
+                e.response.json().get("errorCode", "") == 160048
+                and e.response.status_code == 404
+                and util.is_interactive()
             ):
-                secret_input = getpass.getpass(prompt="\nCould not find secret, enter value here to create it: ")
+                secret_input = getpass.getpass(
+                    prompt="\nCould not find secret, enter value here to create it: "
+                )
                 return self.create_secret(name, secret_input).value
             else:
                 raise e
 
-    def create_secret(self, name: str, value: str, project: str = None) -> secret.Secret:
+    def create_secret(
+        self, name: str, value: str, project: str = None
+    ) -> secret.Secret:
         """Create a new secret.
 
         ```python

python/hopsworks/engine/execution_engine.py

@@ -14,13 +14,13 @@
 #   limitations under the License.
 #
 
-from hopsworks.core import dataset_api, execution_api
-import os
 import logging
+import os
 import time
 import uuid
 
 from hopsworks.client.exceptions import JobExecutionException, RestAPIError
+from hopsworks.core import dataset_api, execution_api
 
 
 class ExecutionEngine:

python/hopsworks/engine/git_engine.py

@@ -14,10 +14,11 @@
 #   limitations under the License.
 #
 
-from hopsworks.core import git_op_execution_api
-from hopsworks.client.exceptions import GitException
-import time
 import logging
+import time
+
+from hopsworks.client.exceptions import GitException
+from hopsworks.core import git_op_execution_api
 
 
 class GitEngine:

python/hopsworks/environment.py

@@ -133,16 +133,18 @@ def install_wheel(self, path: str, await_installation: Optional[bool] = True):
             "packageSource": "WHEEL",
         }
 
-        library_rest = self._library_api._install(
-            library_name, self.name, library_spec
-        )
+        library_rest = self._library_api._install(library_name, self.name, library_spec)
 
         if await_installation:
-            return self._environment_engine.await_library_command(self.name, library_name)
+            return self._environment_engine.await_library_command(
+                self.name, library_name
+            )
 
         return library_rest
 
-    def install_requirements(self, path: str, await_installation: Optional[bool] = True):
+    def install_requirements(
+        self, path: str, await_installation: Optional[bool] = True
+    ):
         """Install libraries specified in a requirements.txt file
 
         ```python
@@ -184,12 +186,12 @@ def install_requirements(self, path: str, await_installation: Optional[bool] = T
             "packageSource": "REQUIREMENTS_TXT",
         }
 
-        library_rest = self._library_api._install(
-            library_name, self.name, library_spec
-        )
+        library_rest = self._library_api._install(library_name, self.name, library_spec)
 
         if await_installation:
-            return self._environment_engine.await_library_command(self.name, library_name)
+            return self._environment_engine.await_library_command(
+                self.name, library_name
+            )
 
         return library_rest
 

python/hopsworks/flink_cluster.py

@@ -15,10 +15,10 @@
 #
 
 import time
-from hopsworks.engine import execution_engine
-from hopsworks.core import execution_api
-from hopsworks.core import flink_cluster_api
+
 from hopsworks import util
+from hopsworks.core import execution_api, flink_cluster_api
+from hopsworks.engine import execution_engine
 
 
 class FlinkCluster:

python/hopsworks/git_commit.py

@@ -14,10 +14,11 @@
 #   limitations under the License.
 #
 
-from hopsworks import util
-import humps
 import json
 
+import humps
+from hopsworks import util
+
 
 class GitCommit:
     def __init__(

python/hopsworks/git_file_status.py

@@ -14,9 +14,9 @@
 #   limitations under the License.
 #
 
-import humps
 import json
 
+import humps
 from hopsworks import util