Skip to content

Commit 073e544

Browse files
artem-astafevartem-astafev
authored
Adding files to deploy VisualQnA application on ROCm vLLM (#1751)
Signed-off-by: Artem Astafev <a.astafev@datamonsters.com>
1 parent e8cdf7d commit 073e544

File tree

10 files changed

+703
-81
lines changed

10 files changed

+703
-81
lines changed

‎VisualQnA/assets/img/visualqna-ui-result-page.png

395 KB
Loading

‎VisualQnA/assets/img/visualqna-ui-starting-page.png

248 KB
Loading

‎VisualQnA/docker_compose/amd/gpu/rocm/README.md

Lines changed: 340 additions & 77 deletions
Large diffs are not rendered by default.

‎VisualQnA/docker_compose/amd/gpu/rocm/compose.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ services:
1515
HUGGINGFACEHUB_API_TOKEN: ${VISUALQNA_HUGGINGFACEHUB_API_TOKEN}
1616
HUGGING_FACE_HUB_TOKEN: ${VISUALQNA_HUGGINGFACEHUB_API_TOKEN}
1717
volumes:
18-
- "/var/opea/visualqna-service/data:/data"
18+
- "${MODEL_CACHE:-./data}:/data"
1919
shm_size: 64g
2020
devices:
2121
- /dev/kfd:/dev/kfd

‎VisualQnA/docker_compose/amd/gpu/rocm/compose_vllm.yaml

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
# Copyright (C) 2024 Advanced Micro Devices, Inc.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
services:
5+
visualqna-vllm-service:
6+
image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}
7+
container_name: visualqna-vllm-service
8+
ports:
9+
- "${VISUALQNA_VLLM_SERVICE_PORT:-8081}:8011"
10+
environment:
11+
no_proxy: ${no_proxy}
12+
http_proxy: ${http_proxy}
13+
https_proxy: ${https_proxy}
14+
HUGGINGFACEHUB_API_TOKEN: ${VISUALQNA_HUGGINGFACEHUB_API_TOKEN}
15+
HF_TOKEN: ${VISUALQNA_HUGGINGFACEHUB_API_TOKEN}
16+
HF_HUB_DISABLE_PROGRESS_BARS: 1
17+
HF_HUB_ENABLE_HF_TRANSFER: 0
18+
WILM_USE_TRITON_FLASH_ATTENTION: 0
19+
PYTORCH_JIT: 0
20+
volumes:
21+
- "${MODEL_CACHE:-./data}:/data"
22+
shm_size: 20G
23+
devices:
24+
- /dev/kfd:/dev/kfd
25+
- /dev/dri/:/dev/dri/
26+
cap_add:
27+
- SYS_PTRACE
28+
group_add:
29+
- video
30+
security_opt:
31+
- seccomp:unconfined
32+
- apparmor=unconfined
33+
command: "--model ${VISUALQNA_LVM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 1 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
34+
ipc: host
35+
lvm:
36+
image: ${REGISTRY:-opea}/lvm:${TAG:-latest}
37+
container_name: lvm-server
38+
depends_on:
39+
- visualqna-vllm-service
40+
ports:
41+
- "9399:9399"
42+
ipc: host
43+
environment:
44+
no_proxy: ${no_proxy}
45+
http_proxy: ${http_proxy}
46+
https_proxy: ${https_proxy}
47+
LVM_COMPONENT_NAME: "OPEA_VLLM_LVM"
48+
LVM_ENDPOINT: ${LVM_ENDPOINT}
49+
LLM_MODEL_ID: ${VISUALQNA_LVM_MODEL_ID}
50+
HF_HUB_DISABLE_PROGRESS_BARS: 1
51+
HF_HUB_ENABLE_HF_TRANSFER: 0
52+
restart: unless-stopped
53+
visualqna-rocm-backend-server:
54+
image: ${REGISTRY:-opea}/visualqna:${TAG:-latest}
55+
container_name: visualqna-rocm-backend-server
56+
depends_on:
57+
- visualqna-vllm-service
58+
- lvm
59+
ports:
60+
- "${BACKEND_SERVICE_PORT:-8888}:8888"
61+
environment:
62+
- no_proxy=${no_proxy}
63+
- https_proxy=${https_proxy}
64+
- http_proxy=${http_proxy}
65+
- MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
66+
- LVM_SERVICE_HOST_IP=${LVM_SERVICE_HOST_IP}
67+
ipc: host
68+
restart: always
69+
visualqna-rocm-ui-server:
70+
image: ${REGISTRY:-opea}/visualqna-ui:${TAG:-latest}
71+
container_name: visualqna-rocm-ui-server
72+
depends_on:
73+
- visualqna-rocm-backend-server
74+
ports:
75+
- "${FRONTEND_SERVICE_PORT:-5173}:5173"
76+
environment:
77+
- no_proxy=${no_proxy}
78+
- https_proxy=${https_proxy}
79+
- http_proxy=${http_proxy}
80+
- BACKEND_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
81+
ipc: host
82+
restart: always
83+
visualqna-nginx-server:
84+
image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
85+
container_name: visualqna-rocm-nginx-server
86+
depends_on:
87+
- visualqna-rocm-backend-server
88+
- visualqna-rocm-ui-server
89+
ports:
90+
- "${NGINX_PORT:-80}:80"
91+
environment:
92+
- no_proxy=${no_proxy}
93+
- https_proxy=${https_proxy}
94+
- http_proxy=${http_proxy}
95+
- FRONTEND_SERVICE_IP=${HOST_IP}
96+
- FRONTEND_SERVICE_PORT=${FRONTEND_SERVICE_PORT}
97+
- BACKEND_SERVICE_NAME=${BACKEND_SERVICE_NAME}
98+
- BACKEND_SERVICE_IP=${HOST_IP}
99+
- BACKEND_SERVICE_PORT=${BACKEND_SERVICE_PORT}
100+
ipc: host
101+
restart: always
102+
103+
networks:
104+
default:
105+
driver: bridge

‎VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
# Copyright (C) 2024 Advanced Micro Devices, Inc
44
# SPDX-License-Identifier: Apache-2.0
55

6-
export HOST_IP=${Your_host_ip_address}
6+
export HOST_IP=${host_ip}
7+
export EXTERNAL_HOST_IP=${host_ip}
78
export VISUALQNA_TGI_SERVICE_PORT="8399"
89
export VISUALQNA_HUGGINGFACEHUB_API_TOKEN=${Your_HUGGINGFACEHUB_API_TOKEN}
910
export VISUALQNA_CARD_ID="card1"

‎VisualQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (C) 2024 Advanced Micro Devices, Inc
4+
# SPDX-License-Identifier: Apache-2.0
5+
6+
export HOST_IP=${host_ip}
7+
export EXTERNAL_HOST_IP=${host_ip}
8+
export VISUALQNA_VLLM_SERVICE_PORT="8081"
9+
export VISUALQNA_HUGGINGFACEHUB_API_TOKEN=${Your_HUGGINGFACEHUB_API_TOKEN}
10+
export VISUALQNA_CARD_ID="card1"
11+
export VISUALQNA_RENDER_ID="renderD136"
12+
export VISUALQNA_LVM_MODEL_ID="Xkev/Llama-3.2V-11B-cot"
13+
export LVM_ENDPOINT="http://${HOST_IP}:${VISUALQNA_VLLM_SERVICE_PORT}"
14+
export LVM_SERVICE_PORT=9399
15+
export MEGA_SERVICE_HOST_IP=${HOST_IP}
16+
export LVM_SERVICE_HOST_IP=${HOST_IP}
17+
export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:${BACKEND_SERVICE_PORT}/v1/visualqna"
18+
export FRONTEND_SERVICE_IP=${HOST_IP}
19+
export FRONTEND_SERVICE_PORT=18001
20+
export BACKEND_SERVICE_NAME=visualqna
21+
export BACKEND_SERVICE_IP=${HOST_IP}
22+
export BACKEND_SERVICE_PORT=18002
23+
export NGINX_PORT=18003

‎VisualQnA/docker_image_build/build.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,8 @@ services:
2929
dockerfile: comps/third_parties/nginx/src/Dockerfile
3030
extends: visualqna
3131
image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
32+
vllm-rocm:
33+
build:
34+
context: GenAIComps
35+
dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu
36+
image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}

‎VisualQnA/tests/test_compose_on_rocm.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ export BACKEND_SERVICE_IP=${HOST_IP}
3333
export BACKEND_SERVICE_PORT=8888
3434
export NGINX_PORT=18003
3535
export PATH="~/miniconda3/bin:$PATH"
36+
export MODEL_CACHE=${model_cache:-"/var/opea/multimodalqna-service/data"}
3637

3738
function build_docker_images() {
3839
opea_branch=${opea_branch:-"main"}
@@ -63,11 +64,11 @@ function start_services() {
6364
sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
6465

6566
# Start Docker Containers
66-
docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
67+
docker compose -f compose.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
6768

6869
n=0
6970
until [[ "$n" -ge 100 ]]; do
70-
docker logs visualqna-tgi-service > ${LOG_PATH}/lvm_tgi_service_start.log
71+
docker logs visualqna-tgi-service >& ${LOG_PATH}/lvm_tgi_service_start.log
7172
if grep -q Connected ${LOG_PATH}/lvm_tgi_service_start.log; then
7273
break
7374
fi

0 commit comments

Comments
 (0)