Skip to content

Commit 062f0f0

Browse files
artem-astafevartem-astafev
authored
Merge branch 'main' into feature/MultimodalQnA_vLLM
2 parents b8cc224 + 073e544 commit 062f0f0

File tree

54 files changed

+2915
-967
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+2915
-967
lines changed

‎.github/workflows/_run-docker-compose.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,11 @@ jobs:
115115
- name: Clean up Working Directory
116116
run: |
117117
sudo rm -rf ${{github.workspace}}/* || true
118+
119+
# clean up containers use ports
120+
cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 5000 | awk -F' : ' '{print $1}')
121+
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
122+
118123
docker system prune -f
119124
docker rmi $(docker images --filter reference="*/*/*:latest" -q) || true
120125
docker rmi $(docker images --filter reference="*/*:ci" -q) || true
@@ -175,6 +180,11 @@ jobs:
175180
export test_case=${{ matrix.test_case }}
176181
export hardware=${{ inputs.hardware }}
177182
bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "containers"
183+
184+
# clean up containers use ports
185+
cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 5000 | awk -F' : ' '{print $1}')
186+
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
187+
178188
docker system prune -f
179189
docker rmi $(docker images --filter reference="*:5000/*/*" -q) || true
180190

‎.github/workflows/nightly-docker-build-publish.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ jobs:
6161

6262
publish:
6363
needs: [get-build-matrix, get-image-list, build-and-test]
64-
if: ${{ needs.get-image-list.outputs.matrix != '' }}
64+
if: always() && ${{ needs.get-image-list.outputs.matrix != '' }}
6565
strategy:
6666
matrix:
6767
image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}

‎.github/workflows/pr-link-path-scan.yml

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,7 @@ jobs:
7676
cd ${{github.workspace}}
7777
fail="FALSE"
7878
repo_name=${{ github.event.pull_request.head.repo.full_name }}
79-
if [ "$(echo "$repo_name"|cut -d'/' -f1)" != "opea-project" ]; then
80-
owner=$(echo "${{ github.event.pull_request.head.repo.full_name }}" |cut -d'/' -f1)
81-
branch="https://github.com/$owner/GenAIExamples/tree/${{ github.event.pull_request.head.ref }}"
82-
else
83-
branch="https://github.com/opea-project/GenAIExamples/blob/${{ github.event.pull_request.head.ref }}"
84-
fi
85-
link_head="https://github.com/opea-project/GenAIExamples/blob/main"
79+
branch="https://github.com/$repo_name/blob/${{ github.event.pull_request.head.ref }}"
8680
8781
merged_commit=$(git log -1 --format='%H')
8882
changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"

‎AudioQnA/tests/test_compose_multilang_on_xeon.sh

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/bin/bash
2+
# Copyright (C) 2025 Intel Corporation
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -e
6+
IMAGE_REPO=${IMAGE_REPO:-"opea"}
7+
IMAGE_TAG=${IMAGE_TAG:-"latest"}
8+
echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
9+
echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
10+
export REGISTRY=${IMAGE_REPO}
11+
export TAG=${IMAGE_TAG}
12+
export MODEL_CACHE=${model_cache:-"./data"}
13+
14+
WORKPATH=$(dirname "$PWD")
15+
LOG_PATH="$WORKPATH/tests"
16+
ip_address=$(hostname -I | awk '{print $1}')
17+
18+
function build_docker_images() {
19+
opea_branch=${opea_branch:-"main"}
20+
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21+
if [[ "${opea_branch}" != "main" ]]; then
22+
cd $WORKPATH
23+
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24+
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25+
find . -type f -name "Dockerfile*" | while read -r file; do
26+
echo "Processing file: $file"
27+
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28+
done
29+
fi
30+
31+
cd $WORKPATH/docker_image_build
32+
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
33+
34+
git clone https://github.com/vllm-project/vllm.git
35+
cd ./vllm/
36+
VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )"
37+
echo "Check out vLLM tag ${VLLM_VER}"
38+
git checkout ${VLLM_VER} &> /dev/null && cd ../
39+
40+
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
41+
service_list="audioqna-multilang audioqna-ui whisper gpt-sovits vllm"
42+
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
43+
44+
docker images && sleep 1s
45+
}
46+
47+
function start_services() {
48+
cd $WORKPATH/docker_compose/intel/cpu/xeon/
49+
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
50+
export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
51+
52+
export MEGA_SERVICE_HOST_IP=${ip_address}
53+
export WHISPER_SERVER_HOST_IP=${ip_address}
54+
export GPT_SOVITS_SERVER_HOST_IP=${ip_address}
55+
export LLM_SERVER_HOST_IP=${ip_address}
56+
57+
export WHISPER_SERVER_PORT=7066
58+
export GPT_SOVITS_SERVER_PORT=9880
59+
export LLM_SERVER_PORT=3006
60+
61+
export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
62+
export host_ip=${ip_address}
63+
64+
# sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
65+
66+
# Start Docker Containers
67+
docker compose -f compose_multilang.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
68+
n=0
69+
until [[ "$n" -ge 200 ]]; do
70+
docker logs vllm-service > $LOG_PATH/vllm_service_start.log 2>&1
71+
if grep -q complete $LOG_PATH/vllm_service_start.log; then
72+
break
73+
fi
74+
sleep 5s
75+
n=$((n+1))
76+
done
77+
}
78+
79+
80+
function validate_megaservice() {
81+
response=$(http_proxy="" curl http://${ip_address}:3008/v1/audioqna -XPOST -d '{"audio": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA", "max_tokens":64}' -H 'Content-Type: application/json')
82+
# always print the log
83+
docker logs whisper-service > $LOG_PATH/whisper-service.log
84+
docker logs gpt-sovits-service > $LOG_PATH/tts-service.log
85+
docker logs vllm-service > $LOG_PATH/vllm-service.log
86+
docker logs audioqna-xeon-backend-server > $LOG_PATH/audioqna-xeon-backend-server.log
87+
echo "$response" | sed 's/^"//;s/"$//' | base64 -d > speech.mp3
88+
89+
if [[ $(file speech.mp3) == *"RIFF"* ]]; then
90+
echo "Result correct."
91+
else
92+
echo "Result wrong."
93+
exit 1
94+
fi
95+
96+
}
97+
98+
99+
function stop_docker() {
100+
cd $WORKPATH/docker_compose/intel/cpu/xeon/
101+
docker compose -f compose_multilang.yaml stop && docker compose rm -f
102+
}
103+
104+
function main() {
105+
106+
stop_docker
107+
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
108+
start_services
109+
110+
validate_megaservice
111+
112+
stop_docker
113+
echo y | docker system prune
114+
115+
}
116+
117+
main

‎ChatQnA/Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
5-
FROM opea/comps-base:$BASE_TAG
6+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
67

78
COPY ./chatqna.py $HOME/chatqna.py
89
COPY ./entrypoint.sh $HOME/entrypoint.sh

‎ChatQnA/README.md

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,15 @@ RAG bridges the knowledge gap by dynamically fetching relevant information from
2323
| Azure | 5th Gen Intel Xeon with Intel AMX | Work-in-progress | Work-in-progress |
2424
| Intel Tiber AI Cloud | 5th Gen Intel Xeon with Intel AMX | Work-in-progress | Work-in-progress |
2525

26-
## Automated Deployment to Ubuntu based system(if not using Terraform) using Intel® Optimized Cloud Modules for **Ansible**
26+
## Automated Deployment to Ubuntu based system (if not using Terraform) using Intel® Optimized Cloud Modules for **Ansible**
2727

2828
To deploy to existing Xeon Ubuntu based system, use our Intel Optimized Cloud Modules for Ansible. This is the same Ansible playbook used by Terraform.
2929
Use this if you are not using Terraform and have provisioned your system with another tool or manually including bare metal.
30-
| Operating System | Intel Optimized Cloud Module for Ansible |
31-
|------------------|------------------------------------------|
32-
| Ubuntu 20.04 | [ChatQnA Ansible Module](https://github.com/intel/optimized-cloud-recipes/tree/main/recipes/ai-opea-chatqna-xeon) |
33-
| Ubuntu 22.04 | Work-in-progress |
30+
31+
| Operating System | Intel Optimized Cloud Module for Ansible |
32+
| ---------------- | ----------------------------------------------------------------------------------------------------------------- |
33+
| Ubuntu 20.04 | [ChatQnA Ansible Module](https://github.com/intel/optimized-cloud-recipes/tree/main/recipes/ai-opea-chatqna-xeon) |
34+
| Ubuntu 22.04 | Work-in-progress |
3435

3536
## Manually Deploy ChatQnA Service
3637

@@ -48,7 +49,7 @@ Note:
4849

4950
1. If you do not have docker installed you can run this script to install docker : `bash docker_compose/install_docker.sh`.
5051

51-
2. The default LLM is `meta-llama/Meta-Llama-3-8B-Instruct`. Before deploying the application, please make sure either you've requested and been granted the access to it on [Huggingface](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) or you've downloaded the model locally from [ModelScope](https://www.modelscope.cn/models).
52+
2. The default LLM is `meta-llama/Meta-Llama-3-8B-Instruct`. Before deploying the application, please make sure either you've requested and been granted the access to it on [Huggingface](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) `or` you've downloaded the model locally from [ModelScope](https://www.modelscope.cn/models).
5253

5354
### Quick Start: 1.Setup Environment Variable
5455

@@ -221,13 +222,14 @@ This ChatQnA use case performs RAG using LangChain, Redis VectorDB and Text Gene
221222
In the below, we provide a table that describes for each microservice component in the ChatQnA architecture, the default configuration of the open source project, hardware, port, and endpoint.
222223

223224
Gaudi default compose.yaml
224-
| MicroService | Open Source Project | HW | Port | Endpoint |
225+
226+
| MicroService | Open Source Project | HW | Port | Endpoint |
225227
| ------------ | ------------------- | ----- | ---- | -------------------- |
226-
| Embedding | Langchain | Xeon | 6000 | /v1/embeddings |
227-
| Retriever | Langchain, Redis | Xeon | 7000 | /v1/retrieval |
228-
| Reranking | Langchain, TEI | Gaudi | 8000 | /v1/reranking |
229-
| LLM | Langchain, TGI | Gaudi | 9000 | /v1/chat/completions |
230-
| Dataprep | Redis, Langchain | Xeon | 6007 | /v1/dataprep/ingest |
228+
| Embedding | Langchain | Xeon | 6000 | /v1/embeddings |
229+
| Retriever | Langchain, Redis | Xeon | 7000 | /v1/retrieval |
230+
| Reranking | Langchain, TEI | Gaudi | 8000 | /v1/reranking |
231+
| LLM | Langchain, TGI | Gaudi | 9000 | /v1/chat/completions |
232+
| Dataprep | Redis, Langchain | Xeon | 6007 | /v1/dataprep/ingest |
231233

232234
### Required Models
233235

‎ChatQnA/assets/img/ui-result-page-faqgen.png

31.9 KB
Loading

‎ChatQnA/assets/img/ui-result-page.png

36.2 KB
Loading

‎ChatQnA/assets/img/ui-starting-page.png

4.63 KB
Loading

0 commit comments

Comments
 (0)