Skip to content

Commit 5090052

Browse files
kennethmhckennethmhc
authored
Merge branch 'main' into HWORKS-1552
2 parents 1378fca + ce8d1fe commit 5090052

13 files changed

+606
-79
lines changed

‎.github/ci.Jenkinsfile

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
def WORKFLOW_RUN_ID = "0"
2+
def SHORT_SHA = ""
3+
def REF_LOADTEST_BRANCH = ""
4+
def WORKFLOW_RUN_URL = ""
5+
6+
pipeline("E2E workflows") {
7+
agent {
8+
label "local"
9+
}
10+
environment {
11+
GITHUB_TOKEN = credentials('990f5312-cd08-48ec-baf8-3b27ff551204')
12+
}
13+
stages {
14+
stage('Clone repository') {
15+
steps {
16+
checkout scm
17+
}
18+
}
19+
stage('Input parameters') {
20+
steps {
21+
script {
22+
SHORT_SHA = sh(script: "git rev-parse --short HEAD", returnStdout: true).trim()
23+
echo "Short sha: ${SHORT_SHA}"
24+
sh "bash .github/workflow_inputs.sh ${SHORT_SHA}"
25+
REF_LOADTEST_BRANCH = sh(script: "cat inputs.json | jq -r '.ref'", returnStdout: true).trim()
26+
echo "Ref loadtest branch: ${REF_LOADTEST_BRANCH}"
27+
}
28+
}
29+
}
30+
stage('Post webhook') {
31+
steps {
32+
script {
33+
def dispatch_response = sh(script: """curl -L -X POST -H "Accept: application/vnd.github+json" \
34+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
35+
-H "X-GitHub-Api-Version: 2022-11-28" \
36+
-d @inputs.json \
37+
https://api.github.com/repos/logicalclocks/loadtest/actions/workflows/e2e_small.yaml/dispatches""",
38+
returnStdout: true
39+
).trim()
40+
echo "Dispatch response: ${dispatch_response}"
41+
sh "rm inputs.json"
42+
}
43+
}
44+
}
45+
stage ('Find workflow run id') {
46+
steps {
47+
script {
48+
sleep 5
49+
TIME_AFTER_WORKFLOW_DISPATCH = sh(script: "date -u +%Y-%m-%dT%H:%M:%SZ", returnStdout: true).trim()
50+
WORKFLOW_RUN_ID = sh(script: """curl -L -X GET -G -H "Accept: application/vnd.github+json" \
51+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
52+
-H "X-GitHub-Api-Version: 2022-11-28" \
53+
-d "event=workflow_dispatch" -d "actor=HopsworksJenkins" -d "branch=${REF_LOADTEST_BRANCH}" \
54+
https://api.github.com/repos/logicalclocks/loadtest/actions/runs | jq -r '.workflow_runs[0].id'""", returnStdout: true).trim()
55+
echo "Workflow run id: ${WORKFLOW_RUN_ID}"
56+
}
57+
}
58+
}
59+
stage('Wait for github action workflow to complete') {
60+
steps {
61+
script {
62+
def status = "in_progress"
63+
while (status == "in_progress" || status == "queued") {
64+
sleep 60
65+
status = sh(script: """curl -L -X GET -H "Accept: application/vnd.github+json" \
66+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
67+
-H "X-GitHub-Api-Version: 2022-11-28" \
68+
https://api.github.com/repos/logicalclocks/loadtest/actions/runs/${WORKFLOW_RUN_ID} | jq -r '.status' """, returnStdout: true).trim()
69+
echo "Status: ${status}"
70+
}
71+
}
72+
}
73+
}
74+
stage('Download artifacts') {
75+
steps {
76+
script {
77+
def REPORT_URL = sh(
78+
script: """curl -L -H "Accept: application/vnd.github+json" \
79+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
80+
-H "X-GitHub-Api-Version: 2022-11-28" \
81+
https://api.github.com/repos/logicalclocks/loadtest/actions/runs/${WORKFLOW_RUN_ID}/artifacts \
82+
| jq -r '.artifacts[] | select(.name == "results_${WORKFLOW_RUN_ID}.xml") | .archive_download_url' """,
83+
returnStdout: true
84+
).trim()
85+
echo "Report url: ${REPORT_URL}"
86+
sh(
87+
script: """curl -L -H \"Accept: application/vnd.github+json\" \
88+
-H \"Authorization: Bearer ${GITHUB_TOKEN}\" \
89+
-H \"X-GitHub-Api-Version: 2022-11-28\" \
90+
-o results.zip "${REPORT_URL}" """
91+
)
92+
sh """if [ -f results.xml ]; then rm results.xml; fi && unzip results.zip && rm results.zip"""
93+
}
94+
}
95+
}
96+
}
97+
post {
98+
always {
99+
junit 'results.xml'
100+
}
101+
}
102+
}

‎.github/workflow_inputs.sh

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
set -e
3+
4+
SHORT_SHA=$1
5+
echo "" > inputs.yaml
6+
7+
8+
if [[ ${ghprbPullTitle} =~ (FSTORE-[0-9]+) || ${ghprbPullTitle} =~ (HWORKS-[0-9]+) ]]; then
9+
captured_string=${BASH_REMATCH[1]}
10+
echo "Found JIRA ticket: ${captured_string}, checking for corresponding pr in loadtest repo"
11+
loadtest_prs=$(curl -L -G \
12+
-H "Accept: application/vnd.github+json" \
13+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
14+
-H "X-GitHub-Api-Version: 2022-11-28" \
15+
-d "state=open" \
16+
https://api.github.com/repos/logicalclocks/loadtest/pulls)
17+
18+
loadtest_branch=$(echo "${loadtest_prs}" | jq -r --arg captured_string ${captured_string} '.[] | select(.title | contains($captured_string)) | .head.ref')
19+
minikube_ip=$(echo "${loadtest_prs}" | jq -r --arg captured_string ${captured_string} '.[] | select(.title | contains($captured_string)) | .labels[] | select(.name | contains("10.87.")) | .name')
20+
labels=$(echo "${loadtest_prs}" | jq -r --arg captured_string ${captured_string} '.[] | select(.title | contains($captured_string)) | .labels[] | select(.name | contains("e2e")) | .name' | paste -sd ",")
21+
fi
22+
23+
if [ -z "${loadtest_branch}" ]; then
24+
echo "No corresponding pr found in loadtest repo, using main branch"
25+
loadtest_branch="main"
26+
else
27+
echo "Found loadtest branch: ${loadtest_branch}"
28+
fi
29+
30+
if [ -z "${minikube_ip}" ]; then
31+
echo "No minikube ip found in labels, using default staging cluster"
32+
minikube_ip="stagingmain.devnet.hops.works" # Make it domain name instead of ip
33+
else
34+
echo "Found minikube ip in loadtest PR labels: ${minikube_ip}"
35+
fi
36+
37+
if [ -z "${labels}" ]; then
38+
echo "No labels found, using default e2e_small"
39+
labels="e2e_small"
40+
else
41+
echo "Found labels: ${labels}"
42+
fi
43+
44+
# .ref is the name of the branch where the workflow dispatch will be sent.
45+
yq '.ref = "main"' -i inputs.yaml
46+
47+
yq '.inputs.max_parallel = "5"' -i inputs.yaml
48+
hopsworks_domain=$minikube_ip yq '.inputs.hopsworks_domain = strenv(hopsworks_domain)' -i inputs.yaml
49+
labels=$labels yq '.inputs.labels = strenv(labels)' -i inputs.yaml
50+
hopsworks_api_branch=${ghprbSourceBranch} yq '.inputs.hopsworks_api_branch = strenv(hopsworks_api_branch)' -i inputs.yaml
51+
loadtest_branch=${loadtest_branch} yq '.inputs.loadtest_head_ref = strenv(loadtest_branch)' -i inputs.yaml
52+
short_sha=$SHORT_SHA yq '.inputs.short_sha = strenv(short_sha)' -i inputs.yaml
53+
user_repo_api=${ghprbPullAuthorLogin} yq '.inputs.user_repo_api = strenv(user_repo_api)' -i inputs.yaml
54+
55+
yq -o=json inputs.yaml > inputs.json
56+
cat inputs.json

‎python/hopsworks/__init__.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,12 @@ def _get_cached_api_key_path():
341341

342342
def _prompt_project(valid_connection, project, is_app):
343343
if project is None:
344-
saas_projects = valid_connection._project_api._get_projects()
344+
if is_app:
345+
# On Serverless we filter out projects owned by other users to make sure automatic login
346+
# without a prompt still happens when users add showcase projects created by other users
347+
saas_projects = valid_connection._project_api._get_owned_projects()
348+
else:
349+
saas_projects = valid_connection._project_api._get_projects()
345350
if len(saas_projects) == 0:
346351
if is_app:
347352
raise ProjectException("Could not find any project")

‎python/hopsworks_common/core/dataset_api.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -645,7 +645,7 @@ def list_files(self, path, offset, limit):
645645
return inode_lst["count"], inode.Inode.from_response_json(inode_lst)
646646

647647
@usage.method_logger
648-
def list(self, remote_path, sort_by=None, limit=1000):
648+
def list(self, remote_path, sort_by=None, offset=0, limit=1000):
649649
"""List all files in a directory in datasets.
650650
651651
:param remote_path: path to list
@@ -659,7 +659,7 @@ def list(self, remote_path, sort_by=None, limit=1000):
659659
# they seem to handle paths differently and return different results, which prevents the merge at the moment (2024-09-03), due to the requirement of backwards-compatibility
660660
_client = client.get_instance()
661661
path_params = ["project", _client._project_id, "dataset", remote_path]
662-
query_params = {"action": "listing", "sort_by": sort_by, "limit": limit}
662+
query_params = {"action": "listing", "sort_by": sort_by, "limit": limit, "offset": offset}
663663
headers = {"content-type": "application/json"}
664664
return _client._send_request(
665665
"GET", path_params, headers=headers, query_params=query_params

‎python/hopsworks_common/core/project_api.py

+35-4
Original file line numberDiff line numberDiff line change
@@ -37,19 +37,50 @@ def _exists(self, name: str):
3737
except RestAPIError:
3838
return False
3939

40-
def _get_projects(self):
41-
"""Get all projects accessible by the user.
40+
def _get_owned_projects(self):
41+
"""Get all projects owned by the current user
4242
4343
# Returns
4444
`List[Project]`: List of Project objects
4545
# Raises
46-
`hopsworks.client.exceptions.RestAPIError`: If unable to get the projects
46+
`hopsworks.client.exceptions.RestAPIError`: If unable to get the project teams
47+
"""
48+
project_team_json = self._get_project_teams()
49+
projects = []
50+
if project_team_json:
51+
# This information can be retrieved calling the /users/profile endpoint but is avoided as that
52+
# requires an API key to have the USER scope which is not guaranteed on serverless
53+
# Until there is a better solution this code is used to get the current user_id to check project ownership
54+
current_user_uid = project_team_json[0]['user']['uid']
55+
for project_team in project_team_json:
56+
if project_team["project"]["owner"]["uid"] == current_user_uid:
57+
projects.append(self._get_project(project_team["project"]["name"]))
58+
return projects
59+
60+
61+
def _get_project_teams(self):
62+
"""Get all project teams for this user.
63+
64+
# Returns
65+
`str`: List of Project teams
66+
# Raises
67+
`hopsworks.client.exceptions.RestAPIError`: If unable to get the project teams
4768
"""
4869
_client = client.get_instance()
4970
path_params = [
5071
"project",
5172
]
52-
project_team_json = _client._send_request("GET", path_params)
73+
return _client._send_request("GET", path_params)
74+
75+
def _get_projects(self):
76+
"""Get all projects accessible by the user.
77+
78+
# Returns
79+
`List[Project]`: List of Project objects
80+
# Raises
81+
`hopsworks.client.exceptions.RestAPIError`: If unable to get the projects
82+
"""
83+
project_team_json = self._get_project_teams()
5384
projects = []
5485
for project_team in project_team_json:
5586
projects.append(self._get_project(project_team["project"]["name"]))

0 commit comments

Comments
 (0)