Skip to content

Commit d91679b

Browse files
vatjvatj
authoredMay 6, 2024
[FSTORE-1371][FSTORE-1346][FSTORE-1212] Migrate to pyproject, ruff and upgrade docs (#200)
* Update dependencies * Fix contributing.md * Fix workflows * Fix contributing.md * Add pyproject, move to ruff, delete manifest.in * Clean up setup and move metadata to pyproject.toml * Freschen up the Readme * Add doc upgrade * Fix mkdocs * Add upper bound to the python version
1 parent a88ea27 commit d91679b

20 files changed

+329
-212
lines changed
 

‎.github/workflows/mkdocs-main.yml

+4-5
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,21 @@ jobs:
77
runs-on: ubuntu-latest
88

99
steps:
10-
- uses: actions/checkout@v2
10+
- uses: actions/checkout@v4
1111
with:
1212
fetch-depth: 0
1313

1414
- name: set dev version
1515
working-directory: ./java
1616
run: echo "DEV_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -Ev 'Download|INFO|WARNING')" >> $GITHUB_ENV
1717

18-
- uses: actions/setup-python@v2
18+
- uses: actions/setup-python@v5
1919
with:
20-
python-version: '3.8'
20+
python-version: "3.10"
2121

2222
- name: install deps
2323
working-directory: ./python
24-
run: cp ../README.md . && pip3 install 'git+https://github.com/logicalclocks/feature-store-api@master#egg=hsfs[python]&subdirectory=python' && pip3 install -e .[dev,docs]
24+
run: cp ../README.md . && pip3 install 'hsfs[python] @ git+https://github.com/logicalclocks/feature-store-api@master#subdirectory=python' && pip3 install -e .[dev,docs]
2525

2626
- name: generate autodoc
2727
run: python3 auto_doc.py
@@ -33,4 +33,3 @@ jobs:
3333
3434
- name: mike deploy docs
3535
run: mike deploy ${{ env.DEV_VERSION }} dev -u
36-

‎.github/workflows/mkdocs-release.yml

+5-9
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,14 @@ name: mkdocs-release
22

33
on:
44
push:
5-
branches: [ branch-* ]
5+
branches: [branch-*]
66

77
jobs:
88
publish-release:
99
runs-on: ubuntu-latest
1010

1111
steps:
12-
- uses: actions/checkout@v2
12+
- uses: actions/checkout@v4
1313
with:
1414
fetch-depth: 0
1515

@@ -20,16 +20,13 @@ jobs:
2020
- name: set major/minor release version
2121
run: echo "MAJOR_VERSION=$(echo $RELEASE_VERSION | sed 's/^\([0-9]*\.[0-9]*\).*$/\1/')" >> $GITHUB_ENV
2222

23-
- uses: actions/setup-python@v2
23+
- uses: actions/setup-python@v5
2424
with:
25-
python-version: '3.8'
25+
python-version: "3.10"
2626

2727
- name: install deps
2828
working-directory: ./python
29-
run: cp ../README.md . && pip3 install pip==22.0.3 && pip3 install -e .[dev,docs]
30-
31-
- name: use dev mike
32-
run: pip3 uninstall -y mike && pip3 install git+'https://github.com/jimporter/mike.git'
29+
run: cp ../README.md . && pip3 install -e .[dev,docs]
3330

3431
- name: generate autodoc
3532
run: python3 auto_doc.py
@@ -43,4 +40,3 @@ jobs:
4340
run: |
4441
mike deploy ${{ env.RELEASE_VERSION }} ${{ env.MAJOR_VERSION }} -u --push
4542
mike alias ${{ env.RELEASE_VERSION }} latest -u --push
46-

‎.github/workflows/python-lint.yml

+33-15
Original file line numberDiff line numberDiff line change
@@ -11,21 +11,39 @@ jobs:
1111
runs-on: ubuntu-latest
1212

1313
steps:
14-
- uses: actions/checkout@v3
15-
- uses: actions/setup-python@v4
16-
with:
17-
python-version: '3.8'
18-
- name: install deps
19-
run: pip install flake8==3.9.0 black==22.3.0 pre-commit-hooks==2.4.0
20-
21-
- name: black
22-
run: black --check python
14+
- uses: actions/checkout@v4
2315

24-
- name: flake8
25-
run: flake8 --config python/.flake8 python
16+
- uses: actions/setup-python@v5
17+
with:
18+
python-version: "3.11"
2619

27-
- name: trailing-whitespace-fixer
28-
run: trailing-whitespace-fixer $(find python -name "*.py" -type f) || exit 1
20+
- name: Get all changed files
21+
id: get-changed-files
22+
uses: tj-actions/changed-files@v44
23+
with:
24+
files_yaml: |
25+
src:
26+
- 'python/**/*.py'
27+
- '!python/tests/**/*.py'
28+
test:
29+
- 'python/tests/**/*.py'
2930
30-
- name: end-of-file-fixer
31-
run: end-of-file-fixer $(find python -name "*.py" -type f) || exit 1
31+
- name: install deps
32+
run: pip install ruff==0.4.2
33+
34+
- name: ruff on python files
35+
if: steps.get-changed-files.outputs.src_any_changed == 'true'
36+
env:
37+
SRC_ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.src_all_changed_files }}
38+
run: ruff check --output-format=github $SRC_ALL_CHANGED_FILES
39+
40+
- name: ruff on test files
41+
if: steps.get-changed-files.outputs.test_any_changed == 'true'
42+
env:
43+
TEST_ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.test_all_changed_files }}
44+
run: ruff check --output-format=github $TEST_ALL_CHANGED_FILES
45+
46+
- name: ruff format --check $ALL_CHANGED_FILES
47+
env:
48+
ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.all_changed_files }}
49+
run: ruff format $ALL_CHANGED_FILES

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ coverage.xml
5050
*.cover
5151
.hypothesis/
5252
.pytest_cache/
53+
.ruff_cache/
5354

5455
# Translations
5556
*.mo

‎CONTRIBUTING.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ We use `mkdocs` together with `mike` ([for versioning](https://github.com/jimpor
7474
1. Currently we are using our own version of `keras-autodoc`
7575

7676
```bash
77-
pip install git+https://github.com/moritzmeister/keras-autodoc@split-tags-properties
77+
pip install git+https://github.com/logicalclocks/keras-autodoc
7878
```
7979

8080
2. Install HOPSWORKS with `docs` extras:

‎README.md

+74-20
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@
99
src="https://img.shields.io/badge/docs-HOPSWORKS-orange"
1010
alt="Hopsworks Documentation"
1111
/></a>
12+
<a><img
13+
src="https://img.shields.io/badge/python-3.8+-blue"
14+
alt="python"
15+
/></a>
1216
<a href="https://pypi.org/project/hopsworks/"><img
1317
src="https://img.shields.io/pypi/v/hopsworks?color=blue"
1418
alt="PyPiStatus"
@@ -17,48 +21,98 @@
1721
src="https://pepy.tech/badge/hopsworks/month"
1822
alt="Downloads"
1923
/></a>
20-
<a href="https://github.com/psf/black"><img
21-
src="https://img.shields.io/badge/code%20style-black-000000.svg"
22-
alt="CodeStyle"
24+
<a href=https://github.com/astral-sh/ruff><img
25+
src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json"
26+
alt="Ruff"
2327
/></a>
2428
<a><img
2529
src="https://img.shields.io/pypi/l/hopsworks?color=green"
2630
alt="License"
2731
/></a>
2832
</p>
2933

30-
*hopsworks* is the python API for interacting with a Hopsworks cluster.
34+
*hopsworks* is the python API for interacting with a Hopsworks cluster. Don't have a Hopsworks cluster just yet? Register an account on [Hopsworks Serverless](https://app.hopsworks.ai/) and get started for free. Once connected to your project, you can:
35+
- Insert dataframes into the online or offline Store, create training datasets or *serve real-time* feature vectors in the Feature Store via the [Feature Store API](https://github.com/logicalclocks/feature-store-api). Already have data somewhere you want to import, checkout our [Storage Connectors](https://docs.hopsworks.ai/latest/user_guides/fs/storage_connector/) documentation.
36+
- register ML models in the model registry and *deploy* them via model serving via the [Machine Learning API](https://gitub.com/logicalclocks/machine-learning-api).
37+
- manage environments, executions, kafka topics and more once you deploy your own Hopsworks cluster, either on-prem or in the cloud. Hopsworks is open-source and has its own [Community Edition](https://github.com/logicalclocks/hopsworks).
3138

32-
## Getting Started On Hopsworks
39+
Our [tutorials](https://github.com/logicalclocks/hopsworks-tutorials) cover a wide range of use cases and example of what *you* can build using Hopsworks.
3340

34-
Instantiate a connection and get the project object
35-
```python
36-
import hopsworks
41+
## Getting Started On Hopsworks
3742

38-
connection = hopsworks.connection()
43+
Once you created a project on [Hopsworks Serverless](https://app.hopsworks.ai) and created a new [Api Key](https://docs.hopsworks.ai/latest/user_guides/projects/api_key/create_api_key/), just use your favourite virtualenv and package manager to install the library:
3944

40-
project = connection.get_project("my_project")
45+
```bash
46+
pip install hopsworks
47+
```
4148

49+
Fire up a notebook and connect to your project, you will be prompted to enter your newly created API key:
50+
```python
51+
import hopsworks
4252

53+
project = hopsworks.login()
4354
```
4455

45-
Create a new project
56+
Access the Feature Store of your project to use as a central repository for your feature data. Use *your* favourite data engineering library (pandas, polars, Spark, etc...) to insert data into the Feature Store, create training datasets or serve real-time feature vectors. Want to predict likelyhood of e-scooter accidents in real-time? Here's how you can do it:
57+
4658
```python
47-
project = connection.create_project("my_project")
59+
fs = project.get_feature_store()
60+
61+
# Write to Feature Groups
62+
bike_ride_fg = fs.get_or_create_feature_group(
63+
name="bike_rides",
64+
version=1,
65+
primary_key=["ride_id"],
66+
event_time="activation_time",
67+
online_enabled=True,
68+
)
69+
70+
fg.insert(bike_rides_df)
71+
72+
# Read from Feature Views
73+
profile_fg = fs.get_feature_group("user_profile", version=1)
74+
75+
bike_ride_fv = fs.get_or_create_feature_view(
76+
name="bike_rides_view",
77+
version=1,
78+
query=bike_ride_fg.select_except(["ride_id"]).join(profile_fg.select(["age", "has_license"]), on="user_id")
79+
)
80+
81+
bike_rides_Q1_2021_df = bike_ride_fv.get_batch_data(
82+
start_date="2021-01-01",
83+
end_date="2021-01-31"
84+
)
85+
86+
# Create a training dataset
87+
version, job = bike_ride_fv.create_train_test_split(
88+
test_size=0.2,
89+
description='Description of a dataset',
90+
# you can have different data formats such as csv, tsv, tfrecord, parquet and others
91+
data_format='csv'
92+
)
93+
94+
# Predict the probability of accident in real-time using new data + context data
95+
bike_ride_fv.init_serving()
96+
97+
while True:
98+
new_ride_vector = poll_ride_queue()
99+
feature_vector = bike_ride_fv.get_online_feature_vector(
100+
{"user_id": new_ride_vector["user_id"]},
101+
passed_features=new_ride_vector
102+
)
103+
accident_probability = model.predict(feature_vector)
48104
```
49105

50-
Upload data to a project
106+
Or you can use the Machine Learning API to register models and deploy them for serving:
51107
```python
52-
dataset_api = project.get_dataset_api()
53-
54-
dataset_api.upload("data.csv", "Resources")
108+
mr = project.get_model_registry()
109+
# or
110+
ms = project.get_model_serving()
55111
```
56112

113+
## Tutorials
57114

58-
59-
60-
61-
You can find more examples on how to use the library in our [hops-examples](https://github.com/logicalclocks/hops-examples) repository.
115+
Need more inspiration or want to learn more about the Hopsworks platform? Check out our [tutorials](https://github.com/logicalclocks/hopsworks-tutorials).
62116

63117
## Documentation
64118

‎docs/CONTRIBUTING.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ We use `mkdocs` together with `mike` ([for versioning](https://github.com/jimpor
7474
1. Currently we are using our own version of `keras-autodoc`
7575

7676
```bash
77-
pip install git+https://github.com/moritzmeister/keras-autodoc@split-tags-properties
77+
pip install git+https://github.com/logicalclocks/keras-autodoc
7878
```
7979

8080
2. Install HOPSWORKS with `docs` extras:

‎docs/css/custom.css

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
:root {
1+
[data-md-color-scheme="hopsworks"] {
22
--md-primary-fg-color: #1EB382;
33
--md-secondary-fg-color: #188a64;
44
--md-tertiary-fg-color: #0d493550;
@@ -24,6 +24,11 @@
2424
box-shadow: 0 0 0 0;
2525
}
2626

27+
.md-tabs__item {
28+
min-width: 2.25rem;
29+
min-height: 1.5rem;
30+
}
31+
2732
.md-tabs__item:hover {
2833
background-color: var(--md-tertiary-fg-color);
2934
transition: background-color 450ms;

‎docs/css/dropdown.css

+9-12
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/* Style The Dropdown Button */
2-
.dropbtn-api {
2+
.dropbtn {
33
color: white;
44
border: none;
55
cursor: pointer;
@@ -9,20 +9,17 @@
99
contain: inherit;
1010
}
1111
.md-tabs {
12-
overflow: inherit;
13-
}
14-
.md-header {
15-
z-index: 1000 !important;
12+
overflow: inherit;
1613
}
1714

1815
/* The container <div> - needed to position the dropdown content */
19-
.dropdown-api {
20-
position: relative;
16+
.dropdown {
17+
position: absolute;
2118
display: inline-block;
2219
}
2320

2421
/* Dropdown Content (Hidden by Default) */
25-
.dropdown-content-api {
22+
.dropdown-content {
2623
display:none;
2724
font-size: 13px;
2825
position: absolute;
@@ -35,21 +32,21 @@ overflow: inherit;
3532
}
3633

3734
/* Links inside the dropdown */
38-
.dropdown-content-api a {
35+
.dropdown-content a {
3936
color: black;
4037
padding: 12px 16px;
4138
text-decoration: none;
4239
display: block;
4340
}
4441

4542
/* Change color of dropdown links on hover */
46-
.dropdown-content-api a:hover {background-color: #f1f1f1}
43+
.dropdown-content a:hover {background-color: #f1f1f1}
4744

4845
/* Show the dropdown menu on hover */
49-
.dropdown-api:hover .dropdown-content-api {
46+
.dropdown:hover .dropdown-content {
5047
display: block;
5148
}
5249

5350
/* Change the background color of the dropdown button when the dropdown content is shown */
54-
.dropdown-api:hover .dropbtn-api {
51+
.dropdown:hover .dropbtn {
5552
}

‎docs/js/dropdown.js

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
document.getElementsByClassName("md-tabs__link")[7].style.display = "none";
2+
document.getElementsByClassName("md-tabs__link")[9].style.display = "none";

‎docs/js/inject-api-links.js

+7-2
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
window.addEventListener("DOMContentLoaded", function () {
22
var windowPathNameSplits = window.location.pathname.split("/");
33
var majorVersionRegex = new RegExp("(\\d+[.]\\d+)")
4+
var latestRegex = new RegExp("latest");
45
if (majorVersionRegex.test(windowPathNameSplits[1])) { // On landing page docs.hopsworks.api/3.0 - URL contains major version
56
// Version API dropdown
67
document.getElementById("hopsworks_api_link").href = "https://docs.hopsworks.ai/hopsworks-api/" + windowPathNameSplits[1] + "/generated/api/login/";
78
document.getElementById("hsfs_api_link").href = "https://docs.hopsworks.ai/feature-store-api/" + windowPathNameSplits[1] + "/generated/api/connection_api/";
89
document.getElementById("hsml_api_link").href = "https://docs.hopsworks.ai/machine-learning-api/" + windowPathNameSplits[1] + "/generated/connection_api/";
910
} else { // on docs.hopsworks.api/feature-store-api/3.0 / docs.hopsworks.api/hopsworks-api/3.0 / docs.hopsworks.api/machine-learning-api/3.0
10-
var apiVersion = windowPathNameSplits[2];
11-
var majorVersion = apiVersion.match(majorVersionRegex)[0];
11+
if (latestRegex.test(windowPathNameSplits[2]) || latestRegex.test(windowPathNameSplits[1])) {
12+
var majorVersion = "latest";
13+
} else {
14+
var apiVersion = windowPathNameSplits[2];
15+
var majorVersion = apiVersion.match(majorVersionRegex)[0];
16+
}
1217
// Version main navigation
1318
document.getElementsByClassName("md-tabs__link")[0].href = "https://docs.hopsworks.ai/" + majorVersion;
1419
document.getElementsByClassName("md-tabs__link")[1].href = "https://colab.research.google.com/github/logicalclocks/hopsworks-tutorials/blob/master/quickstart.ipynb";

0 commit comments

Comments
 (0)
Failed to load comments.