Skip to content

Commit 78d1cd2

Browse files
averseyaversey
committedAug 27, 2024
Automate management of aliases
1 parent 6265269 commit 78d1cd2

File tree

4 files changed

+307
-0
lines changed

4 files changed

+307
-0
lines changed
 

‎.github/workflows/python.yml

+3
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ jobs:
5353
${{ steps.get-changed-files.outputs.all_changed_files }}
5454
run: ruff format $ALL_CHANGED_FILES
5555

56+
- name: check aliases
57+
run: python python/aliases.py check
58+
5659
unit_tests:
5760
name: Unit Tests
5861
needs: lint_stylecheck

‎python/aliases.py

+167
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
#!/usr/bin/python
2+
3+
#
4+
# Copyright 2024 Hopsworks AB
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
"""Scripts for automatic management of aliases."""
20+
21+
import importlib
22+
import sys
23+
from pathlib import Path
24+
25+
26+
SOURCES = [
27+
"hopsworks/__init__.py",
28+
"hopsworks/connection.py",
29+
"hopsworks/internal",
30+
"hopsworks/platform",
31+
"hopsworks/fs",
32+
"hopsworks/ml",
33+
]
34+
IGNORED = ["tests", "hsfs", "hopsworks", "hsml", "hopsworks_common"]
35+
# Everything that is not a top-level file, a part of sources, or a part of ignored is considered to be autmoatically managed.
36+
37+
38+
def traverse(path, f):
39+
if not path.exists():
40+
return
41+
if path.is_file():
42+
f(path)
43+
return
44+
for child in path.iterdir():
45+
traverse(child, f)
46+
47+
48+
def collect_imports(root):
49+
imports = []
50+
51+
def imports_add(file):
52+
pkg = str(file.parent.relative_to(root)).replace("/", ".")
53+
if file.name == "__init__.py":
54+
imports.append(pkg)
55+
elif file.name.endswith(".py"):
56+
imports.append(pkg + "." + file.name[:-3])
57+
58+
for source in SOURCES:
59+
traverse(root / source, imports_add)
60+
61+
return imports
62+
63+
64+
def collect_aliases(root):
65+
for import_str in collect_imports(root):
66+
importlib.import_module(import_str, package=".")
67+
aliases = importlib.import_module("hopsworks.internal.aliases", package=".")
68+
return aliases._aliases
69+
70+
71+
def collect_managed(root):
72+
managed = {}
73+
for pkg, from_imports in collect_aliases(root).items():
74+
pkg = root / pkg.replace(".", "/") / "__init__.py"
75+
managed[pkg] = (
76+
"# ruff: noqa\n"
77+
"# This file is generated by aliases.py. Do not edit it manually!\n"
78+
)
79+
from_imports.sort() # this is needed for determinism
80+
for f, i in from_imports:
81+
managed[pkg] += f"from {f} import {i}\n"
82+
return managed
83+
84+
85+
def fix(root):
86+
managed = collect_managed(root)
87+
for filepath, content in managed.items():
88+
filepath.parent.mkdir(parents=True, exist_ok=True)
89+
filepath.touch()
90+
filepath.write_text(content)
91+
ignored = [root / path for path in SOURCES + IGNORED]
92+
93+
def remove_if_excess(path):
94+
if path.parent == root:
95+
return
96+
if any(path.is_relative_to(p) for p in ignored):
97+
return
98+
if path not in managed:
99+
path.unlink()
100+
101+
traverse(root, remove_if_excess)
102+
103+
104+
def check(root):
105+
global ok
106+
ok = True
107+
managed = collect_managed(root)
108+
ignored = [root / path for path in SOURCES + IGNORED]
109+
110+
def check_file(path):
111+
global ok
112+
if path.parent == root or any(path.is_relative_to(p) for p in ignored):
113+
return
114+
if path not in managed:
115+
print(f"Error: {path} shouldn't exist.")
116+
ok = False
117+
return
118+
if path.read_text() != managed[path]:
119+
print(f"Error: {path} has wrong content.")
120+
ok = False
121+
122+
traverse(root, check_file)
123+
124+
if ok:
125+
print("The aliases are correct!")
126+
else:
127+
print("To fix the errors, run `aliases.py fix`.")
128+
exit(1)
129+
130+
131+
def help(msg=None):
132+
if msg:
133+
print(msg + "\n")
134+
print("Use `aliases.py fix [path]` or `aliases.py check [path]`.")
135+
print(
136+
"`path` is optional, current directory (or its `python` subdirectory) is used by default; it should be the directory containing the hopsworks package, e.g., `./python/`."
137+
)
138+
exit(1)
139+
140+
141+
def main():
142+
if len(sys.argv) == 3:
143+
root = Path(sys.argv[2])
144+
elif len(sys.argv) == 2:
145+
root = Path()
146+
if not (root / "hopsworks").exists():
147+
root = root / "python"
148+
else:
149+
help("Wrong number of arguments.")
150+
151+
root = root.resolve()
152+
if not (root / "hopsworks").exists():
153+
help("The used path doesn't contain the hopsworks package.")
154+
155+
cmd = sys.argv[1]
156+
if cmd in ["f", "fix"]:
157+
cmd = fix
158+
elif cmd in ["c", "check"]:
159+
cmd = check
160+
else:
161+
help("Unknown command.")
162+
163+
cmd(root)
164+
165+
166+
if __name__ == "__main__":
167+
main()

‎python/hopsworks/internal/aliases.py

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#
2+
# Copyright 2024 Hopsworks AB
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
"""Automatic management of aliases.
18+
19+
The associated scripts are located in `python/aliases.py`.
20+
"""
21+
22+
from __future__ import annotations
23+
24+
import functools
25+
import inspect
26+
import warnings
27+
from typing import Optional, Tuple
28+
29+
from hopsworks.internal.exceptions import InternalError
30+
31+
32+
_aliases = {}
33+
34+
35+
def _aliases_add(from_import: Tuple[str, str], *paths: str):
36+
global _aliases
37+
if "." in from_import[1]:
38+
raise InternalError("Impossible to create alias for not importable symbol.")
39+
for p in paths:
40+
_aliases.setdefault(p, []).append(from_import)
41+
42+
43+
def public(*paths: str):
44+
"""Make a function or class publically available.
45+
46+
If you want to publish a constant, use `publish`.
47+
Note that it is impossible to create an alias for a variable, i.e., it is impossible to make a change of a variable in one module to propogate to another variable in another module.
48+
49+
# Arguments
50+
paths: the import paths under which the entity is publically avilable; effectively results in generation of aliases in all of the paths for the entity.
51+
"""
52+
53+
global publics
54+
55+
def decorator(symbol):
56+
if not hasattr(symbol, "__qualname__"):
57+
raise InternalError("The symbol should be importable to be public.")
58+
_aliases_add((symbol.__module__, symbol.__qualname__), *paths)
59+
return symbol
60+
61+
return decorator
62+
63+
64+
def publish(name: str, *paths: str):
65+
"""Make a constant publically available.
66+
67+
Since `public` decorator works only for classes and functions, this function should be used for public constants.
68+
Note that it is impossible to create an alias for a variable, i.e., it is impossible to make a change of a variable in one module to propogate to another variable in another module.
69+
70+
# Arguments
71+
name: name of the constant to be published.
72+
paths: the import paths under which the names declared in the current module will be publically available; effectively results in generation of aliases in all of the paths for all the names declared in the current module.
73+
"""
74+
75+
caller = inspect.getmodule(inspect.stack()[1][0])
76+
77+
_aliases_add((caller.__name__, name), *paths)
78+
79+
80+
class DeprecatedCallWarning(Warning):
81+
pass
82+
83+
84+
def deprecated(*, available_until: Optional[str] = None):
85+
"""Mark a function or class as deprecated.
86+
87+
Use of the entity outside hopsworks will print a warning, saying that it is going to be removed from the public API in one of the future releases.
88+
89+
# Arguments
90+
available_until: the first hopsworks release in which the entity will become unavailable, defaults to `None`; if the release is known, it is reoprted to the external user in the warning.
91+
"""
92+
93+
v = f"version {available_until}" if available_until else "a future release"
94+
95+
def decorator(symbol):
96+
if inspect.isclass(symbol):
97+
98+
@functools.wraps(symbol)
99+
def decorated_f(*args, **kwargs):
100+
caller = inspect.getmodule(inspect.stack()[1][0])
101+
if not caller or not caller.__name__.startswith("hopsworks"):
102+
warnings.warn(
103+
f"Use of {symbol.__qualname__} is deprecated."
104+
f"The function will be removed in {v} of hopsworks.",
105+
DeprecatedCallWarning,
106+
stacklevel=2,
107+
)
108+
return symbol(*args, **kwargs)
109+
110+
return decorated_f
111+
112+
return decorator

‎python/hopsworks/internal/exceptions.py

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#
2+
# Copyright 2024 Hopsworks AB
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
18+
class InternalError(Exception):
19+
"""Internal hopsworks exception.
20+
21+
This is raised in cases when the user of hopsworks cannot be blaimed for the error.
22+
Ideally, this exception should never happen, as it means that one of the hopsworks contributors commited an error.
23+
24+
For example, this exception can be thrown if an internally called function which works only with `str` was given a `float`, or if a public alias is requested for a method of a class.
25+
"""

0 commit comments

Comments
 (0)