Skip to content

Commit 7075739

Browse files
averseyaversey
committedAug 27, 2024
Automate management of aliases
1 parent 6265269 commit 7075739

File tree

4 files changed

+298
-0
lines changed

4 files changed

+298
-0
lines changed
 

‎.github/workflows/python.yml

+3
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ jobs:
5353
${{ steps.get-changed-files.outputs.all_changed_files }}
5454
run: ruff format $ALL_CHANGED_FILES
5555

56+
- name: check aliases
57+
run: python ./python/aliases.py check
58+
5659
unit_tests:
5760
name: Unit Tests
5861
needs: lint_stylecheck

‎python/aliases.py

+158
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/python
2+
3+
#
4+
# Copyright 2024 Hopsworks AB
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
"""Scripts for automatic management of aliases."""
20+
21+
import importlib
22+
import sys
23+
from pathlib import Path
24+
25+
26+
SOURCES = [
27+
"hopsworks/__init__.py",
28+
"hopsworks/connection.py",
29+
"hopsworks/internal",
30+
"hopsworks/platform",
31+
"hopsworks/fs",
32+
"hopsworks/ml",
33+
]
34+
IGNORED = ["tests", "hsfs", "hopsworks", "hsml", "hopsworks_common"]
35+
# Everything that is not a top-level file, a part of sources, or a part of ignored is considered to be autmoatically managed.
36+
37+
38+
def collect_imports(root):
39+
imports = []
40+
41+
def imports_add(file):
42+
pkg = str(file.parent.relative_to(root)).replace("/", ".")
43+
if file.name == "__init__.py":
44+
imports.append(pkg)
45+
elif file.name.endswith(".py"):
46+
imports.append(pkg + "." + file.name[:-3])
47+
48+
for source in SOURCES:
49+
if (root / source).is_file():
50+
imports_add(root / source)
51+
continue
52+
for dirpath, _, filenames in (root / source).walk():
53+
for filename in filenames:
54+
imports_add(dirpath / filename)
55+
return imports
56+
57+
58+
def collect_aliases(root):
59+
for import_str in collect_imports(root):
60+
importlib.import_module(import_str, package=".")
61+
aliases = importlib.import_module("hopsworks.internal.aliases", package=".")
62+
return aliases._aliases
63+
64+
65+
def collect_managed(root):
66+
managed = {}
67+
for pkg, from_imports in collect_aliases(root).items():
68+
pkg = root / pkg.replace(".", "/") / "__init__.py"
69+
managed[pkg] = (
70+
"# ruff: noqa\n"
71+
"# This file is generated by aliases.py. Do not edit it manually!\n"
72+
)
73+
from_imports.sort() # this is needed for determinism
74+
for f, i in from_imports:
75+
managed[pkg] += f"from {f} import {i}\n"
76+
return managed
77+
78+
79+
def fix(root):
80+
managed = collect_managed(root)
81+
for filepath, content in managed.items():
82+
filepath.parent.mkdir(parents=True, exist_ok=True)
83+
filepath.touch()
84+
filepath.write_text(content)
85+
ignored = [root / path for path in SOURCES + IGNORED]
86+
for dirpath, _, filenames in root.walk():
87+
if dirpath == root:
88+
continue
89+
for filename in filenames:
90+
filepath = dirpath / filename
91+
if any(filepath.is_relative_to(p) for p in ignored):
92+
continue
93+
if filepath not in managed:
94+
filepath.unlink()
95+
96+
97+
def check(root):
98+
ok = True
99+
managed = collect_managed(root)
100+
ignored = [root / path for path in SOURCES + IGNORED]
101+
for dirpath, _, filenames in root.walk():
102+
if dirpath == root:
103+
continue
104+
for filename in filenames:
105+
filepath = dirpath / filename
106+
if any(filepath.is_relative_to(p) for p in ignored):
107+
continue
108+
if filepath not in managed:
109+
print(f"Error: {filepath} shouldn't exist.")
110+
ok = False
111+
continue
112+
if filepath.read_text() != managed[filepath]:
113+
print(f"Error: {filepath} has wrong content.")
114+
ok = False
115+
if ok:
116+
print("The aliases are correct!")
117+
else:
118+
print("To fix the errors, run `aliases.py fix`.")
119+
exit(1)
120+
121+
122+
def help(msg=None):
123+
if msg:
124+
print(msg + "\n")
125+
print("Use `aliases.py fix [path]` or `aliases.py check [path]`.")
126+
print(
127+
"`path` is optional, current directory (or its `python` subdirectory) is used by default; it should be the directory containing the hopsworks package, e.g., `./python/`."
128+
)
129+
exit(1)
130+
131+
132+
def main():
133+
if len(sys.argv) == 3:
134+
root = Path(sys.argv[2])
135+
elif len(sys.argv) == 2:
136+
root = Path()
137+
if not (root / "hopsworks").exists():
138+
root = root / "python"
139+
else:
140+
help("Wrong number of arguments.")
141+
142+
root = root.resolve()
143+
if not (root / "hopsworks").exists():
144+
help("The used path doesn't contain the hopsworks package.")
145+
146+
cmd = sys.argv[1]
147+
if cmd in ["f", "fix"]:
148+
cmd = fix
149+
elif cmd in ["c", "check"]:
150+
cmd = check
151+
else:
152+
help("Unknown command.")
153+
154+
cmd(root)
155+
156+
157+
if __name__ == "__main__":
158+
main()

‎python/hopsworks/internal/aliases.py

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#
2+
# Copyright 2024 Hopsworks AB
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
"""Automatic management of aliases.
18+
19+
The associated scripts are located in `python/aliases.py`.
20+
"""
21+
22+
from __future__ import annotations
23+
24+
import functools
25+
import inspect
26+
import warnings
27+
from typing import Optional, Tuple
28+
29+
from hopsworks.internal.exceptions import InternalError
30+
31+
32+
_aliases = {}
33+
34+
35+
def _aliases_add(from_import: Tuple[str, str], *paths: str):
36+
global _aliases
37+
if "." in from_import[1]:
38+
raise InternalError("Impossible to create alias for not importable symbol.")
39+
for p in paths:
40+
_aliases.setdefault(p, []).append(from_import)
41+
42+
43+
def public(*paths: str):
44+
"""Make a function or class publically available.
45+
46+
If you want to publish a constant, use `publish`.
47+
Note that it is impossible to create an alias for a variable, i.e., it is impossible to make a change of a variable in one module to propogate to another variable in another module.
48+
49+
# Arguments
50+
paths: the import paths under which the entity is publically avilable; effectively results in generation of aliases in all of the paths for the entity.
51+
"""
52+
53+
global publics
54+
55+
def decorator(symbol):
56+
if not hasattr(symbol, "__qualname__"):
57+
raise InternalError("The symbol should be importable to be public.")
58+
_aliases_add((symbol.__module__, symbol.__qualname__), *paths)
59+
return symbol
60+
61+
return decorator
62+
63+
64+
def publish(name: str, *paths: str):
65+
"""Make a constant publically available.
66+
67+
Since `public` decorator works only for classes and functions, this function should be used for public constants.
68+
Note that it is impossible to create an alias for a variable, i.e., it is impossible to make a change of a variable in one module to propogate to another variable in another module.
69+
70+
# Arguments
71+
name: name of the constant to be published.
72+
paths: the import paths under which the names declared in the current module will be publically available; effectively results in generation of aliases in all of the paths for all the names declared in the current module.
73+
"""
74+
75+
caller = inspect.getmodule(inspect.stack()[1][0])
76+
77+
_aliases_add((caller.__name__, name), *paths)
78+
79+
80+
class DeprecatedCallWarning(Warning):
81+
pass
82+
83+
84+
def deprecated(*, available_until: Optional[str] = None):
85+
"""Mark a function or class as deprecated.
86+
87+
Use of the entity outside hopsworks will print a warning, saying that it is going to be removed from the public API in one of the future releases.
88+
89+
# Arguments
90+
available_until: the first hopsworks release in which the entity will become unavailable, defaults to `None`; if the release is known, it is reoprted to the external user in the warning.
91+
"""
92+
93+
v = f"version {available_until}" if available_until else "a future release"
94+
95+
def decorator(symbol):
96+
if inspect.isclass(symbol):
97+
98+
@functools.wraps(symbol)
99+
def decorated_f(*args, **kwargs):
100+
caller = inspect.getmodule(inspect.stack()[1][0])
101+
if not caller or not caller.__name__.startswith("hopsworks"):
102+
warnings.warn(
103+
f"Use of {symbol.__qualname__} is deprecated."
104+
f"The function will be removed in {v} of hopsworks.",
105+
DeprecatedCallWarning,
106+
stacklevel=2,
107+
)
108+
return symbol(*args, **kwargs)
109+
110+
return decorated_f
111+
112+
return decorator

‎python/hopsworks/internal/exceptions.py

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#
2+
# Copyright 2024 Hopsworks AB
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
18+
class InternalError(Exception):
19+
"""Internal hopsworks exception.
20+
21+
This is raised in cases when the user of hopsworks cannot be blaimed for the error.
22+
Ideally, this exception should never happen, as it means that one of the hopsworks contributors commited an error.
23+
24+
For example, this exception can be thrown if an internally called function which works only with `str` was given a `float`, or if a public alias is requested for a method of a class.
25+
"""

0 commit comments

Comments
 (0)