Skip to content

Commit c31b2d7

Browse files
bubriksbubriks
authored
Merge branch 'main' into FSTORE-1015
2 parents 46c07e0 + ce8d1fe commit c31b2d7

File tree

97 files changed

+1943
-920
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

97 files changed

+1943
-920
lines changed

‎.github/ci.Jenkinsfile

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
def WORKFLOW_RUN_ID = "0"
2+
def SHORT_SHA = ""
3+
def REF_LOADTEST_BRANCH = ""
4+
def WORKFLOW_RUN_URL = ""
5+
6+
pipeline("E2E workflows") {
7+
agent {
8+
label "local"
9+
}
10+
environment {
11+
GITHUB_TOKEN = credentials('990f5312-cd08-48ec-baf8-3b27ff551204')
12+
}
13+
stages {
14+
stage('Clone repository') {
15+
steps {
16+
checkout scm
17+
}
18+
}
19+
stage('Input parameters') {
20+
steps {
21+
script {
22+
SHORT_SHA = sh(script: "git rev-parse --short HEAD", returnStdout: true).trim()
23+
echo "Short sha: ${SHORT_SHA}"
24+
sh "bash .github/workflow_inputs.sh ${SHORT_SHA}"
25+
REF_LOADTEST_BRANCH = sh(script: "cat inputs.json | jq -r '.ref'", returnStdout: true).trim()
26+
echo "Ref loadtest branch: ${REF_LOADTEST_BRANCH}"
27+
}
28+
}
29+
}
30+
stage('Post webhook') {
31+
steps {
32+
script {
33+
def dispatch_response = sh(script: """curl -L -X POST -H "Accept: application/vnd.github+json" \
34+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
35+
-H "X-GitHub-Api-Version: 2022-11-28" \
36+
-d @inputs.json \
37+
https://api.github.com/repos/logicalclocks/loadtest/actions/workflows/e2e_small.yaml/dispatches""",
38+
returnStdout: true
39+
).trim()
40+
echo "Dispatch response: ${dispatch_response}"
41+
sh "rm inputs.json"
42+
}
43+
}
44+
}
45+
stage ('Find workflow run id') {
46+
steps {
47+
script {
48+
sleep 5
49+
TIME_AFTER_WORKFLOW_DISPATCH = sh(script: "date -u +%Y-%m-%dT%H:%M:%SZ", returnStdout: true).trim()
50+
WORKFLOW_RUN_ID = sh(script: """curl -L -X GET -G -H "Accept: application/vnd.github+json" \
51+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
52+
-H "X-GitHub-Api-Version: 2022-11-28" \
53+
-d "event=workflow_dispatch" -d "actor=HopsworksJenkins" -d "branch=${REF_LOADTEST_BRANCH}" \
54+
https://api.github.com/repos/logicalclocks/loadtest/actions/runs | jq -r '.workflow_runs[0].id'""", returnStdout: true).trim()
55+
echo "Workflow run id: ${WORKFLOW_RUN_ID}"
56+
}
57+
}
58+
}
59+
stage('Wait for github action workflow to complete') {
60+
steps {
61+
script {
62+
def status = "in_progress"
63+
while (status == "in_progress" || status == "queued") {
64+
sleep 60
65+
status = sh(script: """curl -L -X GET -H "Accept: application/vnd.github+json" \
66+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
67+
-H "X-GitHub-Api-Version: 2022-11-28" \
68+
https://api.github.com/repos/logicalclocks/loadtest/actions/runs/${WORKFLOW_RUN_ID} | jq -r '.status' """, returnStdout: true).trim()
69+
echo "Status: ${status}"
70+
}
71+
}
72+
}
73+
}
74+
stage('Download artifacts') {
75+
steps {
76+
script {
77+
def REPORT_URL = sh(
78+
script: """curl -L -H "Accept: application/vnd.github+json" \
79+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
80+
-H "X-GitHub-Api-Version: 2022-11-28" \
81+
https://api.github.com/repos/logicalclocks/loadtest/actions/runs/${WORKFLOW_RUN_ID}/artifacts \
82+
| jq -r '.artifacts[] | select(.name == "results_${WORKFLOW_RUN_ID}.xml") | .archive_download_url' """,
83+
returnStdout: true
84+
).trim()
85+
echo "Report url: ${REPORT_URL}"
86+
sh(
87+
script: """curl -L -H \"Accept: application/vnd.github+json\" \
88+
-H \"Authorization: Bearer ${GITHUB_TOKEN}\" \
89+
-H \"X-GitHub-Api-Version: 2022-11-28\" \
90+
-o results.zip "${REPORT_URL}" """
91+
)
92+
sh """if [ -f results.xml ]; then rm results.xml; fi && unzip results.zip && rm results.zip"""
93+
}
94+
}
95+
}
96+
}
97+
post {
98+
always {
99+
junit 'results.xml'
100+
}
101+
}
102+
}

‎.github/workflow_inputs.sh

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
set -e
3+
4+
SHORT_SHA=$1
5+
echo "" > inputs.yaml
6+
7+
8+
if [[ ${ghprbPullTitle} =~ (FSTORE-[0-9]+) || ${ghprbPullTitle} =~ (HWORKS-[0-9]+) ]]; then
9+
captured_string=${BASH_REMATCH[1]}
10+
echo "Found JIRA ticket: ${captured_string}, checking for corresponding pr in loadtest repo"
11+
loadtest_prs=$(curl -L -G \
12+
-H "Accept: application/vnd.github+json" \
13+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
14+
-H "X-GitHub-Api-Version: 2022-11-28" \
15+
-d "state=open" \
16+
https://api.github.com/repos/logicalclocks/loadtest/pulls)
17+
18+
loadtest_branch=$(echo "${loadtest_prs}" | jq -r --arg captured_string ${captured_string} '.[] | select(.title | contains($captured_string)) | .head.ref')
19+
minikube_ip=$(echo "${loadtest_prs}" | jq -r --arg captured_string ${captured_string} '.[] | select(.title | contains($captured_string)) | .labels[] | select(.name | contains("10.87.")) | .name')
20+
labels=$(echo "${loadtest_prs}" | jq -r --arg captured_string ${captured_string} '.[] | select(.title | contains($captured_string)) | .labels[] | select(.name | contains("e2e")) | .name' | paste -sd ",")
21+
fi
22+
23+
if [ -z "${loadtest_branch}" ]; then
24+
echo "No corresponding pr found in loadtest repo, using main branch"
25+
loadtest_branch="main"
26+
else
27+
echo "Found loadtest branch: ${loadtest_branch}"
28+
fi
29+
30+
if [ -z "${minikube_ip}" ]; then
31+
echo "No minikube ip found in labels, using default staging cluster"
32+
minikube_ip="stagingmain.devnet.hops.works" # Make it domain name instead of ip
33+
else
34+
echo "Found minikube ip in loadtest PR labels: ${minikube_ip}"
35+
fi
36+
37+
if [ -z "${labels}" ]; then
38+
echo "No labels found, using default e2e_small"
39+
labels="e2e_small"
40+
else
41+
echo "Found labels: ${labels}"
42+
fi
43+
44+
# .ref is the name of the branch where the workflow dispatch will be sent.
45+
yq '.ref = "main"' -i inputs.yaml
46+
47+
yq '.inputs.max_parallel = "5"' -i inputs.yaml
48+
hopsworks_domain=$minikube_ip yq '.inputs.hopsworks_domain = strenv(hopsworks_domain)' -i inputs.yaml
49+
labels=$labels yq '.inputs.labels = strenv(labels)' -i inputs.yaml
50+
hopsworks_api_branch=${ghprbSourceBranch} yq '.inputs.hopsworks_api_branch = strenv(hopsworks_api_branch)' -i inputs.yaml
51+
loadtest_branch=${loadtest_branch} yq '.inputs.loadtest_head_ref = strenv(loadtest_branch)' -i inputs.yaml
52+
short_sha=$SHORT_SHA yq '.inputs.short_sha = strenv(short_sha)' -i inputs.yaml
53+
user_repo_api=${ghprbPullAuthorLogin} yq '.inputs.user_repo_api = strenv(user_repo_api)' -i inputs.yaml
54+
55+
yq -o=json inputs.yaml > inputs.json
56+
cat inputs.json

‎.github/workflows/java.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
distribution: "adopt"
2222

2323
- name: Cache local Maven repository
24-
uses: actions/cache@v2
24+
uses: actions/cache@v4
2525
with:
2626
path: ~/.m2/repository
2727
key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}
@@ -50,7 +50,7 @@ jobs:
5050
distribution: "adopt"
5151

5252
- name: Cache local Maven repository
53-
uses: actions/cache@v2
53+
uses: actions/cache@v4
5454
with:
5555
path: ~/.m2/repository
5656
key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}
@@ -79,7 +79,7 @@ jobs:
7979
distribution: "adopt"
8080

8181
- name: Cache local Maven repository
82-
uses: actions/cache@v2
82+
uses: actions/cache@v4
8383
with:
8484
path: ~/.m2/repository
8585
key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}

‎.github/workflows/mkdocs-main.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727
run: python3 ./python/auto_doc.py
2828

2929
- name: Cache local Maven repository
30-
uses: actions/cache@v2
30+
uses: actions/cache@v4
3131
with:
3232
path: ~/.m2/repository
3333
key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}

‎.github/workflows/mkdocs-release.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ jobs:
3232
run: python3 ./python/auto_doc.py
3333

3434
- name: Cache local Maven repository
35-
uses: actions/cache@v2
35+
uses: actions/cache@v4
3636
with:
3737
path: ~/.m2/repository
3838
key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}

‎docs/templates/model-registry/model_api.md

+5-1
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,14 @@
1212

1313
{{ml_create_sl}}
1414

15-
## Creation of a generic model
15+
## Creation of a Python model
1616

1717
{{ml_create_py}}
1818

19+
## Creation of a LLM
20+
21+
{{ml_create_llm}}
22+
1923
## Retrieval
2024

2125
{{ml_get}}

‎docs/templates/model-serving/predictor_state_api.md

-3
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22

33
The state of a deployment corresponds to the state of the predictor configured in it.
44

5-
!!! note
6-
Currently, only one predictor is supported in a deployment. Support for multiple predictors (the inference graphs) is coming soon.
7-
85
## Retrieval
96

107
{{ps_get}}

‎java/beam/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<artifactId>hsfs-parent</artifactId>
77
<groupId>com.logicalclocks</groupId>
8-
<version>4.2.0-SNAPSHOT</version>
8+
<version>4.3.0-SNAPSHOT</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

‎java/flink/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<artifactId>hsfs-parent</artifactId>
77
<groupId>com.logicalclocks</groupId>
8-
<version>4.2.0-SNAPSHOT</version>
8+
<version>4.3.0-SNAPSHOT</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

‎java/hsfs/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<artifactId>hsfs-parent</artifactId>
77
<groupId>com.logicalclocks</groupId>
8-
<version>4.2.0-SNAPSHOT</version>
8+
<version>4.3.0-SNAPSHOT</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

‎java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ public abstract Object getOrCreateFeatureView(String name, T2 query, Integer ver
142142
List<String> labels) throws FeatureStoreException, IOException;
143143

144144
/**
145-
* Get a external feature group object from the feature store.
145+
* Get an external feature group object from the feature store.
146146
*
147147
* @param name the name of the feature group
148148
* @param version the version of the feature group
@@ -154,7 +154,7 @@ public abstract Object getExternalFeatureGroup(@NonNull String name, @NonNull In
154154
throws FeatureStoreException, IOException;
155155

156156
/**
157-
* Get a external feature group object with default version `1` from the feature store.
157+
* Get an external feature group object with default version `1` from the feature store.
158158
*
159159
* @param name the name of the feature group
160160
* @return ExternalFeatureGroup

‎java/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<groupId>com.logicalclocks</groupId>
88
<artifactId>hsfs-parent</artifactId>
99
<packaging>pom</packaging>
10-
<version>4.2.0-SNAPSHOT</version>
10+
<version>4.3.0-SNAPSHOT</version>
1111
<modules>
1212
<module>hsfs</module>
1313
<module>spark</module>

‎java/spark/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
<parent>
2323
<artifactId>hsfs-parent</artifactId>
2424
<groupId>com.logicalclocks</groupId>
25-
<version>4.2.0-SNAPSHOT</version>
25+
<version>4.3.0-SNAPSHOT</version>
2626
</parent>
2727
<modelVersion>4.0.0</modelVersion>
2828

0 commit comments

Comments
 (0)