Skip to content

Commit 7f17aeb

Browse files
averseyaversey
authored
[FSTORE-1481] Move basic things from hopsworks to hopsworks_common (logicalclocks#245)
* Move basic things from hopsworks * Adapt the moved code * Remove user from hopsworks_common * Move user from hsfs * Adapt user.py * Add alias for Command
1 parent dc8367d commit 7f17aeb

17 files changed

+957
-835
lines changed

‎python/hopsworks/command.py

+7-42
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2022 Hopsworks AB
2+
# Copyright 2024 Hopsworks AB
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -14,46 +14,11 @@
1414
# limitations under the License.
1515
#
1616

17-
import humps
17+
from hopsworks_common.command import (
18+
Command,
19+
)
1820

1921

20-
class Command:
21-
def __init__(
22-
self,
23-
id=None,
24-
status=None,
25-
op=None,
26-
install_type=None,
27-
error_message=None,
28-
# path to the custom commands file in the case the op=custom_command
29-
custom_commands_file=None,
30-
args=None,
31-
type=None,
32-
href=None,
33-
count=None,
34-
**kwargs,
35-
):
36-
self._id = id
37-
self._op = op
38-
self._install_type = install_type
39-
self._status = status
40-
self._error_message = error_message
41-
self._count = count
42-
self._custom_commands_file = custom_commands_file
43-
self._args = args
44-
45-
@classmethod
46-
def from_response_json(cls, json_dict):
47-
json_decamelized = humps.decamelize(json_dict)
48-
if "items" in json_decamelized:
49-
return [cls(**command) for command in json_decamelized["items"]]
50-
else:
51-
return []
52-
53-
@property
54-
def status(self):
55-
return self._status
56-
57-
@property
58-
def error_message(self):
59-
return self._error_message
22+
__all__ = [
23+
Command,
24+
]

‎python/hopsworks/core/environment_api.py

+7-141
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2022 Hopsworks AB
2+
# Copyright 2024 Hopsworks AB
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -14,145 +14,11 @@
1414
# limitations under the License.
1515
#
1616

17-
import json
18-
from typing import List, Optional
17+
from hopsworks_common.core.environment_api import (
18+
EnvironmentApi,
19+
)
1920

20-
from hopsworks import client, environment
21-
from hopsworks.engine import environment_engine
2221

23-
24-
class EnvironmentApi:
25-
def __init__(
26-
self,
27-
project_id,
28-
project_name,
29-
):
30-
self._project_id = project_id
31-
self._project_name = project_name
32-
33-
self._environment_engine = environment_engine.EnvironmentEngine(project_id)
34-
35-
def create_environment(
36-
self,
37-
name: str,
38-
description: Optional[str] = None,
39-
base_environment_name: Optional[str] = "python-feature-pipeline",
40-
await_creation: Optional[bool] = True,
41-
) -> environment.Environment:
42-
"""Create Python environment for the project
43-
44-
```python
45-
46-
import hopsworks
47-
48-
project = hopsworks.login()
49-
50-
env_api = project.get_environment_api()
51-
52-
new_env = env_api.create_environment("my_custom_environment", base_environment_name="python-feature-pipeline")
53-
54-
55-
```
56-
# Arguments
57-
name: name of the environment
58-
base_environment_name: the name of the environment to clone from
59-
await_creation: bool. If True the method returns only when the creation is finished. Default True
60-
# Returns
61-
`Environment`: The Environment object
62-
# Raises
63-
`RestAPIError`: If unable to create the environment
64-
"""
65-
_client = client.get_instance()
66-
67-
path_params = [
68-
"project",
69-
self._project_id,
70-
"python",
71-
"environments",
72-
name,
73-
]
74-
headers = {"content-type": "application/json"}
75-
data = {
76-
"name": name,
77-
"baseImage": {"name": base_environment_name, "description": description},
78-
}
79-
env = environment.Environment.from_response_json(
80-
_client._send_request(
81-
"POST", path_params, headers=headers, data=json.dumps(data)
82-
),
83-
self._project_id,
84-
self._project_name,
85-
)
86-
87-
if await_creation:
88-
self._environment_engine.await_environment_command(name)
89-
90-
return env
91-
92-
def get_environments(self) -> List[environment.Environment]:
93-
"""
94-
Get all available python environments in the project
95-
"""
96-
_client = client.get_instance()
97-
98-
path_params = ["project", self._project_id, "python", "environments"]
99-
query_params = {"expand": ["libraries", "commands"]}
100-
headers = {"content-type": "application/json"}
101-
return environment.Environment.from_response_json(
102-
_client._send_request(
103-
"GET", path_params, query_params=query_params, headers=headers
104-
),
105-
self._project_id,
106-
self._project_name,
107-
)
108-
109-
def get_environment(self, name: str) -> environment.Environment:
110-
"""Get handle for the Python environment for the project
111-
112-
```python
113-
114-
import hopsworks
115-
116-
project = hopsworks.login()
117-
118-
env_api = project.get_environment_api()
119-
120-
env = env_api.get_environment("my_custom_environment")
121-
122-
```
123-
# Arguments
124-
name: name of the environment
125-
# Returns
126-
`Environment`: The Environment object
127-
# Raises
128-
`RestAPIError`: If unable to get the environment
129-
"""
130-
_client = client.get_instance()
131-
132-
path_params = ["project", self._project_id, "python", "environments", name]
133-
query_params = {"expand": ["libraries", "commands"]}
134-
headers = {"content-type": "application/json"}
135-
return environment.Environment.from_response_json(
136-
_client._send_request(
137-
"GET", path_params, query_params=query_params, headers=headers
138-
),
139-
self._project_id,
140-
self._project_name,
141-
)
142-
143-
def _delete(self, name):
144-
"""Delete the Python environment.
145-
:param name: name of environment to delete
146-
:type environment: Environment
147-
"""
148-
_client = client.get_instance()
149-
150-
path_params = [
151-
"project",
152-
self._project_id,
153-
"python",
154-
"environments",
155-
name,
156-
]
157-
headers = {"content-type": "application/json"}
158-
(_client._send_request("DELETE", path_params, headers=headers),)
22+
__all__ = [
23+
EnvironmentApi,
24+
]

‎python/hopsworks/core/library_api.py

+7-47
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2022 Hopsworks AB
2+
# Copyright 2024 Hopsworks AB
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -14,51 +14,11 @@
1414
# limitations under the License.
1515
#
1616

17-
import json
17+
from hopsworks_common.core.library_api import (
18+
LibraryApi,
19+
)
1820

19-
from hopsworks import client, library
2021

21-
22-
class LibraryApi:
23-
def __init__(
24-
self,
25-
project_id,
26-
project_name,
27-
):
28-
self._project_id = project_id
29-
self._project_name = project_name
30-
31-
def _install(self, library_name: str, name: str, library_spec: dict):
32-
"""Install a library in the environment
33-
34-
# Arguments
35-
library_name: Name of the library.
36-
name: Name of the environment.
37-
library_spec: installation payload
38-
# Returns
39-
`Library`: The library object
40-
# Raises
41-
`RestAPIError`: If unable to install library
42-
"""
43-
44-
_client = client.get_instance()
45-
46-
path_params = [
47-
"project",
48-
self._project_id,
49-
"python",
50-
"environments",
51-
name,
52-
"libraries",
53-
library_name,
54-
]
55-
56-
headers = {"content-type": "application/json"}
57-
library_rest = library.Library.from_response_json(
58-
_client._send_request(
59-
"POST", path_params, headers=headers, data=json.dumps(library_spec)
60-
),
61-
environment=self,
62-
project_id=self._project_id,
63-
)
64-
return library_rest
22+
__all__ = [
23+
LibraryApi,
24+
]

‎python/hopsworks/engine/environment_engine.py

+7-91
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2022 Hopsworks AB
2+
# Copyright 2024 Hopsworks AB
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -14,95 +14,11 @@
1414
# limitations under the License.
1515
#
1616

17-
import time
17+
from hopsworks_common.engine.environment_engine import (
18+
EnvironmentEngine,
19+
)
1820

19-
from hopsworks import client, command, environment, library
20-
from hopsworks.client.exceptions import EnvironmentException, RestAPIError
2121

22-
23-
class EnvironmentEngine:
24-
def __init__(self, project_id):
25-
self._project_id = project_id
26-
27-
def await_library_command(self, environment_name, library_name):
28-
commands = [command.Command(status="ONGOING")]
29-
while len(commands) > 0 and not self._is_final_status(commands[0]):
30-
time.sleep(5)
31-
library = self._poll_commands_library(environment_name, library_name)
32-
if library is None:
33-
commands = []
34-
else:
35-
commands = library._commands
36-
37-
def await_environment_command(self, environment_name):
38-
commands = [command.Command(status="ONGOING")]
39-
while len(commands) > 0 and not self._is_final_status(commands[0]):
40-
time.sleep(5)
41-
environment = self._poll_commands_environment(environment_name)
42-
if environment is None:
43-
commands = []
44-
else:
45-
commands = environment._commands
46-
47-
def _is_final_status(self, command):
48-
if command.status == "FAILED":
49-
raise EnvironmentException(
50-
"Command failed with stacktrace: \n{}".format(command.error_message)
51-
)
52-
elif command.status == "SUCCESS":
53-
return True
54-
else:
55-
return False
56-
57-
def _poll_commands_library(self, environment_name, library_name):
58-
_client = client.get_instance()
59-
60-
path_params = [
61-
"project",
62-
self._project_id,
63-
"python",
64-
"environments",
65-
environment_name,
66-
"libraries",
67-
library_name,
68-
]
69-
70-
query_params = {"expand": "commands"}
71-
headers = {"content-type": "application/json"}
72-
73-
try:
74-
return library.Library.from_response_json(
75-
_client._send_request(
76-
"GET", path_params, headers=headers, query_params=query_params
77-
),
78-
None,
79-
None,
80-
)
81-
except RestAPIError as e:
82-
if (
83-
e.response.json().get("errorCode", "") == 300003
84-
and e.response.status_code == 404
85-
):
86-
return None
87-
88-
def _poll_commands_environment(self, environment_name):
89-
_client = client.get_instance()
90-
91-
path_params = [
92-
"project",
93-
self._project_id,
94-
"python",
95-
"environments",
96-
environment_name,
97-
]
98-
99-
query_params = {"expand": "commands"}
100-
headers = {"content-type": "application/json"}
101-
102-
return environment.Environment.from_response_json(
103-
_client._send_request(
104-
"GET", path_params, headers=headers, query_params=query_params
105-
),
106-
None,
107-
None,
108-
)
22+
__all__ = [
23+
EnvironmentEngine,
24+
]

0 commit comments

Comments
 (0)