Skip to content

Commit 1e654ee

Browse files
chyundunovDatamonsterschyundunovDatamonsters
committed
Merge branch 'main' of https://github.com/opea-project/GenAIExamples into feature/CodeGen_CodeTrans_k8s
2 parents d93f017 + 162f5a8 commit 1e654ee

File tree

300 files changed

+7563
-2787
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

300 files changed

+7563
-2787
lines changed

‎.github/CODEOWNERS

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
1-
* liang1.lv@intel.com feng.tian@intel.com suyue.chen@intel.com
1+
# Code owners will review PRs within their respective folders.
2+
3+
* liang1.lv@intel.com feng.tian@intel.com suyue.chen@intel.com kaokao.lv@intel.com minmin.hou@intel.com rita.brugarolas.brufau@intel.com
24
/.github/ suyue.chen@intel.com ze.pan@intel.com
3-
/AgentQnA/ kaokao.lv@intel.com minmin.hou@intel.com
5+
/AgentQnA/ abolfazl.shahbazi@intel.com kaokao.lv@intel.com minmin.hou@intel.com
46
/AudioQnA/ sihan.chen@intel.com wenjiao.yue@intel.com
57
/AvatarChatbot/ chun.tao@intel.com kaokao.lv@intel.com
68
/ChatQnA/ liang1.lv@intel.com letong.han@intel.com
7-
/CodeGen/ liang1.lv@intel.com xinyao.wang@intel.com
8-
/CodeTrans/ sihan.chen@intel.com xinyao.wang@intel.com
9+
/CodeGen/ liang1.lv@intel.com
10+
/CodeTrans/ sihan.chen@intel.com
911
/DBQnA/ supriya.krishnamurthi@intel.com liang1.lv@intel.com
10-
/DocIndexRetriever/ kaokao.lv@intel.com chendi.xue@intel.com
11-
/DocSum/ letong.han@intel.com xinyao.wang@intel.com
12+
/DocIndexRetriever/ abolfazl.shahbazi@intel.com kaokao.lv@intel.com chendi.xue@intel.com
13+
/DocSum/ letong.han@intel.com
1214
/EdgeCraftRAG/ yongbo.zhu@intel.com mingyuan.qi@intel.com
13-
/FaqGen/ yogesh.pandey@intel.com xinyao.wang@intel.com
15+
/FinanceAgent/ abolfazl.shahbazi@intel.com kaokao.lv@intel.com minmin.hou@intel.com rita.brugarolas.brufau@intel.com
1416
/GraphRAG/ rita.brugarolas.brufau@intel.com abolfazl.shahbazi@intel.com
1517
/InstructionTuning/ xinyu.ye@intel.com kaokao.lv@intel.com
1618
/MultimodalQnA/ melanie.h.buehler@intel.com tiep.le@intel.com
@@ -19,5 +21,6 @@
1921
/SearchQnA/ sihan.chen@intel.com letong.han@intel.com
2022
/Text2Image/ wenjiao.yue@intel.com xinyu.ye@intel.com
2123
/Translation/ liang1.lv@intel.com sihan.chen@intel.com
22-
/VideoQnA/ huiling.bao@intel.com xinyao.wang@intel.com
23-
/VisualQnA/ liang1.lv@intel.com sihan.chen@intel.com
24+
/VideoQnA/ huiling.bao@intel.com
25+
/VisualQnA/ liang1.lv@intel.com sihan.chen@intel.com
26+
/WorkflowExecAgent/ joshua.jian.ern.liew@intel.com kaokao.lv@intel.com

‎.github/env/_build_image.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Copyright (C) 2025 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
export VLLM_VER=v0.8.3
5+
export VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0

‎.github/workflows/_build_image.yml

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,17 +75,12 @@ jobs:
7575
run: |
7676
cd ${{ github.workspace }}/${{ inputs.example }}/docker_image_build
7777
docker_compose_path=${{ github.workspace }}/${{ inputs.example }}/docker_image_build/build.yaml
78+
source ${{ github.workspace }}/.github/env/_build_image.sh
7879
if [[ $(grep -c "vllm:" ${docker_compose_path}) != 0 ]]; then
79-
git clone https://github.com/vllm-project/vllm.git && cd vllm
80-
VLLM_VER=v0.8.3
81-
echo "Check out vLLM tag ${VLLM_VER}"
82-
git checkout ${VLLM_VER} &> /dev/null && cd ../
80+
git clone -b ${VLLM_VER} --single-branch https://github.com/vllm-project/vllm.git
8381
fi
8482
if [[ $(grep -c "vllm-gaudi:" ${docker_compose_path}) != 0 ]]; then
85-
git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
86-
VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
87-
echo "Check out vLLM tag ${VLLM_VER}"
88-
git checkout ${VLLM_VER} &> /dev/null && cd ../
83+
git clone -b ${VLLM_FORK_VER} --single-branch https://github.com/HabanaAI/vllm-fork.git
8984
fi
9085
git clone --depth 1 --branch ${{ inputs.opea_branch }} https://github.com/opea-project/GenAIComps.git
9186
cd GenAIComps && git rev-parse HEAD && cd ../

‎.github/workflows/_helm-e2e.yml

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
# SPDX-License-Identifier: Apache-2.0
33

44
name: Helm Chart E2e Test For Call
5-
permissions: read-all
5+
permissions:
6+
contents: read
7+
68
on:
79
workflow_call:
810
inputs:
@@ -135,16 +137,28 @@ jobs:
135137
env:
136138
example: ${{ inputs.example }}
137139
run: |
138-
CHART_NAME="${example,,}" # CodeGen
139-
echo "CHART_NAME=$CHART_NAME" >> $GITHUB_ENV
140-
echo "RELEASE_NAME=${CHART_NAME}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
141-
echo "NAMESPACE=${CHART_NAME}-$(head -c 4 /dev/urandom | xxd -p)" >> $GITHUB_ENV
142-
echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
143-
echo "TEST_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
144-
echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
145-
echo "should_cleanup=false" >> $GITHUB_ENV
146-
echo "skip_validate=false" >> $GITHUB_ENV
147-
echo "CHART_FOLDER=${example}/kubernetes/helm" >> $GITHUB_ENV
140+
if [[ ! "$example" =~ ^[a-zA-Z0-9]{1,20}$ ]] || [[ "$example" =~ \.\. ]] || [[ "$example" == -* || "$example" == *- ]]; then
141+
echo "Error: Invalid input - only lowercase alphanumeric and internal hyphens allowed"
142+
exit 1
143+
fi
144+
# SAFE_PREFIX="kb-"
145+
CHART_NAME="${SAFE_PREFIX}$(echo "$example" | tr '[:upper:]' '[:lower:]')"
146+
RAND_SUFFIX=$(openssl rand -hex 2 | tr -dc 'a-f0-9')
147+
148+
cat <<EOF >> $GITHUB_ENV
149+
CHART_NAME=${CHART_NAME}
150+
RELEASE_NAME=${CHART_NAME}-$(date +%s)
151+
NAMESPACE=ns-${CHART_NAME}-${RAND_SUFFIX}
152+
ROLLOUT_TIMEOUT_SECONDS=600s
153+
TEST_TIMEOUT_SECONDS=600s
154+
KUBECTL_TIMEOUT_SECONDS=60s
155+
should_cleanup=false
156+
skip_validate=false
157+
CHART_FOLDER=${example}/kubernetes/helm
158+
EOF
159+
160+
echo "Generated safe variables:" >> $GITHUB_STEP_SUMMARY
161+
echo "- CHART_NAME: ${CHART_NAME}" >> $GITHUB_STEP_SUMMARY
148162
149163
- name: Helm install
150164
id: install

‎.github/workflows/_run-docker-compose.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,10 @@ jobs:
204204
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
205205
206206
echo "Cleaning up images ..."
207+
df -h
208+
sleep 1
209+
docker system df
210+
sleep 1
207211
if [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
208212
docker system prune -a -f
209213
else
@@ -213,7 +217,13 @@ jobs:
213217
docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
214218
docker system prune -f
215219
fi
220+
sleep 5
216221
docker images
222+
sleep 1
223+
df -h
224+
sleep 1
225+
docker system df
226+
sleep 1
217227
218228
- name: Publish pipeline artifact
219229
if: ${{ !cancelled() }}

‎.github/workflows/daily-update-vllm-version.yml

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
# Copyright (C) 2025 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Daily update vLLM & vLLM-fork version
5+
6+
on:
7+
schedule:
8+
- cron: "30 22 * * *"
9+
workflow_dispatch:
10+
11+
env:
12+
BRANCH_NAME: "update"
13+
USER_NAME: "CICD-at-OPEA"
14+
USER_EMAIL: "CICD@opea.dev"
15+
16+
jobs:
17+
freeze-tag:
18+
runs-on: ubuntu-latest
19+
strategy:
20+
matrix:
21+
include:
22+
- repo: vLLM
23+
repo_name: vllm-project/vllm
24+
ver_name: VLLM_VER
25+
- repo: vLLM-fork
26+
repo_name: HabanaAI/vllm-fork
27+
ver_name: VLLM_FORK_VER
28+
fail-fast: false
29+
permissions:
30+
contents: write
31+
pull-requests: write
32+
steps:
33+
- name: Checkout repository
34+
uses: actions/checkout@v4
35+
with:
36+
fetch-depth: 0
37+
ref: ${{ github.ref }}
38+
39+
- name: Set up Git
40+
run: |
41+
git config --global user.name ${{ env.USER_NAME }}
42+
git config --global user.email ${{ env.USER_EMAIL }}
43+
git remote set-url origin https://${{ env.USER_NAME }}:"${{ secrets.ACTION_TOKEN }}"@github.com/${{ github.repository }}.git
44+
git fetch
45+
46+
if git ls-remote https://github.com/${{ github.repository }}.git "refs/heads/${{ env.BRANCH_NAME }}_${{ matrix.repo }}" | grep -q "refs/heads/${{ env.BRANCH_NAME }}_${{ matrix.repo }}"; then
47+
echo "branch ${{ env.BRANCH_NAME }}_${{ matrix.repo }} exists"
48+
git checkout ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
49+
else
50+
echo "branch ${{ env.BRANCH_NAME }}_${{ matrix.repo }} not exists"
51+
git checkout -b ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
52+
git push origin ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
53+
echo "branch ${{ env.BRANCH_NAME }}_${{ matrix.repo }} created successfully"
54+
fi
55+
56+
- name: Run script
57+
run: |
58+
latest_vllm_ver=$(curl -s "https://api.github.com/repos/${{ matrix.repo_name }}/tags" | jq '.[0].name' -)
59+
latest_vllm_ver=$(echo "$latest_vllm_ver" | sed 's/"//g')
60+
echo "latest_vllm_ver=${latest_vllm_ver}" >> "$GITHUB_ENV"
61+
find . -type f -name "*.sh" -exec sed -i "s/${{ matrix.ver_name }}=.*/${{ matrix.ver_name }}=${latest_vllm_ver}/" {} \;
62+
63+
- name: Commit changes
64+
run: |
65+
git add .
66+
if git diff-index --quiet HEAD --; then
67+
echo "No changes detected, skipping commit."
68+
exit 1
69+
else
70+
git commit -s -m "Update ${{ matrix.repo }} version to ${latest_vllm_ver}"
71+
git push --set-upstream origin ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
72+
fi
73+
74+
- name: Create Pull Request
75+
env:
76+
GH_TOKEN: ${{ secrets.ACTION_TOKEN }}
77+
run: |
78+
pr_count=$(curl -H "Authorization: token ${{ secrets.ACTION_TOKEN }}" -s "https://api.github.com/repos/${{ github.repository }}/pulls?state=all&head=${{ env.USER_NAME }}:${{ env.BRANCH_NAME }}_${{ matrix.repo }}" | jq '. | length')
79+
if [ $pr_count -gt 0 ]; then
80+
echo "Pull Request exists"
81+
pr_number=$(curl -H "Authorization: token ${{ secrets.ACTION_TOKEN }}" -s "https://api.github.com/repos/${{ github.repository }}/pulls?state=all&head=${{ env.USER_NAME }}:${{ env.BRANCH_NAME }}_${{ matrix.repo }}" | jq '.[0].number')
82+
gh pr edit ${pr_number} \
83+
--title "Update ${{ matrix.repo }} version to ${latest_vllm_ver}" \
84+
--body "Update ${{ matrix.repo }} version to ${latest_vllm_ver}"
85+
echo "Pull Request updated successfully"
86+
else
87+
echo "Pull Request does not exists..."
88+
gh pr create \
89+
-B main \
90+
-H ${{ env.BRANCH_NAME }}_${{ matrix.repo }} \
91+
--title "Update ${{ matrix.repo }} version to ${latest_vllm_ver}" \
92+
--body "Update ${{ matrix.repo }} version to ${latest_vllm_ver}"
93+
echo "Pull Request created successfully"
94+
fi

‎.github/workflows/daily_check_issue_and_pr.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,4 @@ jobs:
2626
close-pr-message: "This PR was closed because it has been stalled for 7 days with no activity."
2727
repo-token: ${{ secrets.ACTION_TOKEN }}
2828
start-date: "2025-03-01T00:00:00Z"
29+
exempt-issue-labels: "Backlog"

‎.github/workflows/pr-chart-e2e.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ concurrency:
1919
jobs:
2020
job1:
2121
name: Get-Test-Matrix
22+
permissions:
23+
contents: read
24+
pull-requests: read
2225
runs-on: ubuntu-latest
2326
outputs:
2427
run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }}

‎.github/workflows/pr-link-path-scan.yml

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ jobs:
2323
- name: Check the Validity of Hyperlinks
2424
run: |
2525
cd ${{github.workspace}}
26+
delay=1
2627
fail="FALSE"
2728
merged_commit=$(git log -1 --format='%H')
2829
changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"
@@ -35,15 +36,20 @@ jobs:
3536
# echo $url_line
3637
url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//')
3738
path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-)
38-
response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")|| true
39-
if [ "$response" -ne 200 ]; then
40-
echo "**********Validation failed, try again**********"
41-
response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url")
42-
if [ "$response_retry" -eq 200 ]; then
43-
echo "*****Retry successfully*****"
44-
else
45-
echo "Invalid link from ${{github.workspace}}/$path: $url"
46-
fail="TRUE"
39+
if [[ "$url" == "https://platform.openai.com/api-keys"* ]]; then
40+
echo "Link "$url" from ${{github.workspace}}/$path needs to be verified by a real person."
41+
else
42+
sleep $delay
43+
response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")|| true
44+
if [ "$response" -ne 200 ]; then
45+
echo "**********Validation failed ($response), try again**********"
46+
response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url")
47+
if [ "$response_retry" -eq 200 ]; then
48+
echo "*****Retry successfully*****"
49+
else
50+
echo "Invalid link ($response_retry) from ${{github.workspace}}/$path: $url"
51+
fail="TRUE"
52+
fi
4753
fi
4854
fi
4955
done
@@ -74,6 +80,7 @@ jobs:
7480
- name: Checking Relative Path Validity
7581
run: |
7682
cd ${{github.workspace}}
83+
delay=1
7784
fail="FALSE"
7885
repo_name=${{ github.event.pull_request.head.repo.full_name }}
7986
branch="https://github.com/$repo_name/blob/${{ github.event.pull_request.head.ref }}"
@@ -105,14 +112,15 @@ jobs:
105112
if [[ "$png_line" == *#* ]]; then
106113
if [ -n "changed_files" ] && echo "$changed_files" | grep -q "^${refer_path}$"; then
107114
url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIExamples||')$png_path
115+
sleep $delay
108116
response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev")
109117
if [ "$response" -ne 200 ]; then
110-
echo "**********Validation failed, try again**********"
118+
echo "**********Validation failed ($response), try again**********"
111119
response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url_dev")
112120
if [ "$response_retry" -eq 200 ]; then
113121
echo "*****Retry successfully*****"
114122
else
115-
echo "Invalid path from ${{github.workspace}}/$refer_path: $png_path"
123+
echo "Invalid path ($response_retry) from ${{github.workspace}}/$refer_path: $png_path"
116124
fail="TRUE"
117125
fi
118126
else

‎.github/workflows/scorecard.yml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Copyright (C) 2025 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
# This workflow uses actions that are not certified by GitHub. They are provided
5+
# by a third-party and are governed by separate terms of service, privacy
6+
# policy, and support documentation.
7+
8+
name: Scorecard supply-chain security
9+
on:
10+
# For Branch-Protection check. Only the default branch is supported. See
11+
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
12+
branch_protection_rule:
13+
# To guarantee Maintained check is occasionally updated. See
14+
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
15+
schedule:
16+
- cron: '18 7 * * 3'
17+
push:
18+
branches: [ "main" ]
19+
20+
# Declare default permissions as read only.
21+
permissions: read-all
22+
23+
jobs:
24+
analysis:
25+
name: Scorecard analysis
26+
runs-on: ubuntu-latest
27+
# `publish_results: true` only works when run from the default branch. conditional can be removed if disabled.
28+
if: github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request'
29+
permissions:
30+
# Needed to upload the results to code-scanning dashboard.
31+
security-events: write
32+
# Needed to publish results and get a badge (see publish_results below).
33+
id-token: write
34+
# Uncomment the permissions below if installing in a private repository.
35+
# contents: read
36+
# actions: read
37+
38+
steps:
39+
- name: "Checkout code"
40+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
41+
with:
42+
persist-credentials: false
43+
44+
- name: "Run analysis"
45+
uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
46+
with:
47+
results_file: results.sarif
48+
results_format: sarif
49+
# (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
50+
# - you want to enable the Branch-Protection check on a *public* repository, or
51+
# - you are installing Scorecard on a *private* repository
52+
# To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.
53+
repo_token: ${{ secrets.ACTION_TOKEN }}
54+
55+
# Public repositories:
56+
# - Publish results to OpenSSF REST API for easy access by consumers
57+
# - Allows the repository to include the Scorecard badge.
58+
# - See https://github.com/ossf/scorecard-action#publishing-results.
59+
# For private repositories:
60+
# - `publish_results` will always be set to `false`, regardless
61+
# of the value entered here.
62+
publish_results: true
63+
64+
# (Optional) Uncomment file_mode if you have a .gitattributes with files marked export-ignore
65+
# file_mode: git
66+
67+
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
68+
# format to the repository Actions tab.
69+
- name: "Upload artifact"
70+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
71+
with:
72+
name: SARIF file
73+
path: results.sarif
74+
retention-days: 5
75+
76+
# Upload the results to GitHub's code scanning dashboard (optional).
77+
# Commenting out will disable upload of results to your repo's Code Scanning dashboard
78+
- name: "Upload to code-scanning"
79+
uses: github/codeql-action/upload-sarif@v3
80+
with:
81+
sarif_file: results.sarif

0 commit comments

Comments
 (0)