Skip to content

Commit 58f3fa3

Browse files
Chingis YundunovChingis Yundunov
Chingis Yundunov
authored and
Chingis Yundunov
committed
ProductivitySuite - fix Docker compose file, set envs script for deploy app on AMD GPU
Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
1 parent 60871f2 commit 58f3fa3

File tree

2 files changed

+207
-0
lines changed

2 files changed

+207
-0
lines changed

‎ChatQnA/docker_compose/amd/gpu/rocm-vllm/compose_vllm_dev.yaml

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
# Copyright (C) 2024 Advanced Micro Devices, Inc.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
services:
5+
chatqna-redis-vector-db-dev:
6+
image: redis/redis-stack:7.2.0-v9
7+
container_name: chatqna-redis-vector-db-dev
8+
ports:
9+
- "${CHATQNA_REDIS_VECTOR_PORT:-6379}:6379"
10+
- "${CHATQNA_REDIS_VECTOR_INSIGHT_PORT:-8001}:8001"
11+
chatqna-dataprep-redis-service-dev:
12+
image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
13+
container_name: chatqna-dataprep-redis-service-dev
14+
depends_on:
15+
- chatqna-redis-vector-db-dev
16+
- chatqna-tei-embedding-service-dev
17+
ports:
18+
- "${CHATQNA_REDIS_DATAPREP_PORT}:6007"
19+
environment:
20+
no_proxy: ${no_proxy}
21+
http_proxy: ${http_proxy}
22+
https_proxy: ${https_proxy}
23+
REDIS_URL: ${CHATQNA_REDIS_URL}
24+
INDEX_NAME: ${CHATQNA_INDEX_NAME}
25+
TEI_ENDPOINT: ${CHATQNA_TEI_EMBEDDING_ENDPOINT}
26+
HUGGINGFACEHUB_API_TOKEN: ${CHATQNA_HUGGINGFACEHUB_API_TOKEN}
27+
chatqna-tei-embedding-service-dev:
28+
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
29+
container_name: chatqna-tei-embedding-service-dev
30+
ports:
31+
- "${CHATQNA_TEI_EMBEDDING_PORT}:80"
32+
volumes:
33+
- "/var/opea/chatqna-service/data:/data"
34+
shm_size: 1g
35+
ipc: host
36+
environment:
37+
no_proxy: ${no_proxy}
38+
http_proxy: ${http_proxy}
39+
https_proxy: ${https_proxy}
40+
command: --model-id ${CHATQNA_EMBEDDING_MODEL_ID} --auto-truncate
41+
chatqna-retriever-dev:
42+
image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
43+
container_name: chatqna-retriever-dev
44+
depends_on:
45+
- chatqna-redis-vector-db
46+
ports:
47+
- "${CHATQNA_REDIS_RETRIEVER_PORT}:7000"
48+
ipc: host
49+
environment:
50+
no_proxy: ${no_proxy}
51+
http_proxy: ${http_proxy}
52+
https_proxy: ${https_proxy}
53+
REDIS_URL: ${CHATQNA_REDIS_URL}
54+
INDEX_NAME: ${CHATQNA_INDEX_NAME}
55+
TEI_EMBEDDING_ENDPOINT: ${CHATQNA_TEI_EMBEDDING_ENDPOINT}
56+
restart: unless-stopped
57+
chatqna-tei-reranking-service-dev:
58+
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
59+
container_name: chatqna-tei-reranking-service-dev
60+
ports:
61+
- "${CHATQNA_TEI_RERANKING_PORT}:80"
62+
volumes:
63+
- "/var/opea/chatqna-service/data:/data"
64+
shm_size: 1g
65+
environment:
66+
no_proxy: ${no_proxy}
67+
http_proxy: ${http_proxy}
68+
https_proxy: ${https_proxy}
69+
HUGGINGFACEHUB_API_TOKEN: ${CHATQNA_HUGGINGFACEHUB_API_TOKEN}
70+
HF_HUB_DISABLE_PROGRESS_BARS: 1
71+
HF_HUB_ENABLE_HF_TRANSFER: 0
72+
command: --model-id ${CHATQNA_RERANK_MODEL_ID} --auto-truncate
73+
chatqna-vllm-service-dev:
74+
image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
75+
container_name: chatqna-vllm-service-dev
76+
ports:
77+
- "${CHATQNA_VLLM_SERVICE_PORT:-8081}:8011"
78+
environment:
79+
no_proxy: ${no_proxy}
80+
http_proxy: ${http_proxy}
81+
https_proxy: ${https_proxy}
82+
HUGGINGFACEHUB_API_TOKEN: ${CHATQNA_HUGGINGFACEHUB_API_TOKEN}
83+
HF_TOKEN: ${CHATQNA_HUGGINGFACEHUB_API_TOKEN}
84+
HF_HUB_DISABLE_PROGRESS_BARS: 1
85+
HF_HUB_ENABLE_HF_TRANSFER: 0
86+
WILM_USE_TRITON_FLASH_ATTENTION: 0
87+
PYTORCH_JIT: 0
88+
volumes:
89+
- "/var/opea/chatqna-service/data:/data"
90+
shm_size: 128G
91+
devices:
92+
- /dev/kfd:/dev/kfd
93+
- /dev/dri/:/dev/dri/
94+
cap_add:
95+
- SYS_PTRACE
96+
group_add:
97+
- video
98+
security_opt:
99+
- seccomp:unconfined
100+
- apparmor=unconfined
101+
command: "--model ${CHATQNA_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
102+
ipc: host
103+
chatqna-backend-server-dev:
104+
image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
105+
container_name: chatqna-backend-server-dev
106+
depends_on:
107+
- chatqna-redis-vector-db
108+
- chatqna-tei-embedding-service
109+
- chatqna-retriever
110+
- chatqna-tei-reranking-service
111+
- chatqna-vllm-service
112+
ports:
113+
- "${CHATQNA_BACKEND_SERVICE_PORT}:8888"
114+
environment:
115+
no_proxy: ${no_proxy}
116+
https_proxy: ${https_proxy}
117+
http_proxy: ${http_proxy}
118+
MEGA_SERVICE_HOST_IP: ${CHATQNA_MEGA_SERVICE_HOST_IP}
119+
EMBEDDING_SERVER_HOST_IP: ${HOST_IP}
120+
EMBEDDING_SERVER_PORT: ${CHATQNA_TEI_EMBEDDING_PORT:-80}
121+
RETRIEVER_SERVICE_HOST_IP: ${HOST_IP}
122+
RERANK_SERVER_HOST_IP: ${HOST_IP}
123+
RERANK_SERVER_PORT: ${CHATQNA_TEI_RERANKING_PORT:-80}
124+
LLM_SERVER_HOST_IP: ${HOST_IP}
125+
LLM_SERVER_PORT: ${CHATQNA_VLLM_SERVICE_PORT:-80}
126+
LLM_MODEL: ${CHATQNA_LLM_MODEL_ID}
127+
ipc: host
128+
restart: always
129+
chatqna-ui-server-dev:
130+
image: ${REGISTRY:-opea}/chatqna-ui:${TAG:-latest}
131+
container_name: chatqna-ui-server-dev
132+
depends_on:
133+
- chatqna-backend-server
134+
ports:
135+
- "${CHATQNA_FRONTEND_SERVICE_PORT}:5173"
136+
environment:
137+
no_proxy: ${no_proxy}
138+
https_proxy: ${https_proxy}
139+
http_proxy: ${http_proxy}
140+
CHAT_BASE_URL: ${CHATQNA_BACKEND_SERVICE_ENDPOINT}
141+
UPLOAD_FILE_BASE_URL: ${CHATQNA_DATAPREP_SERVICE_ENDPOINT}
142+
GET_FILE: ${CHATQNA_DATAPREP_GET_FILE_ENDPOINT}
143+
DELETE_FILE: ${CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT}
144+
ipc: host
145+
restart: always
146+
chatqna-nginx-server-dev:
147+
image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
148+
container_name: chatqna-nginx-server-dev
149+
depends_on:
150+
- chatqna-backend-server
151+
- chatqna-ui-server
152+
ports:
153+
- "${CHATQNA_NGINX_PORT}:80"
154+
environment:
155+
no_proxy: ${no_proxy}
156+
https_proxy: ${https_proxy}
157+
http_proxy: ${http_proxy}
158+
FRONTEND_SERVICE_IP: ${CHATQNA_FRONTEND_SERVICE_IP}
159+
FRONTEND_SERVICE_PORT: ${CHATQNA_FRONTEND_SERVICE_PORT}
160+
BACKEND_SERVICE_NAME: ${CHATQNA_BACKEND_SERVICE_NAME}
161+
BACKEND_SERVICE_IP: ${CHATQNA_BACKEND_SERVICE_IP}
162+
BACKEND_SERVICE_PORT: ${CHATQNA_BACKEND_SERVICE_PORT}
163+
ipc: host
164+
restart: always
165+
166+
networks:
167+
default:
168+
driver: bridge

‎ChatQnA/docker_compose/amd/gpu/rocm-vllm/set_env_vllm_dev.sh

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (C) 2024 Intel Corporation
4+
# SPDX-License-Identifier: Apache-2.0
5+
6+
# SPDX-License-Identifier: Apache-2.0
7+
8+
export HOST_IP='192.165.1.21'
9+
export HOST_IP_EXTERNAL='direct-supercomputer1.powerml.co'
10+
export CHATQNA_HUGGINGFACEHUB_API_TOKEN="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"
11+
export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
12+
export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base"
13+
export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
14+
export MODEL=${CHATQNA_LLM_MODEL_ID}
15+
export CHATQNA_VLLM_SERVICE_PORT=18008
16+
export CHATQNA_TEI_EMBEDDING_PORT=18090
17+
export CHATQNA_TEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}"
18+
export CHATQNA_TEI_RERANKING_PORT=18808
19+
export CHATQNA_REDIS_VECTOR_PORT=16379
20+
export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001
21+
export CHATQNA_REDIS_DATAPREP_PORT=18103
22+
export CHATQNA_REDIS_RETRIEVER_PORT=17000
23+
export CHATQNA_FRONTEND_SERVICE_IP=${HOST_IP}
24+
export CHATQNA_FRONTEND_SERVICE_PORT=18101
25+
export CHATQNA_BACKEND_SERVICE_NAME=chatqna
26+
export CHATQNA_BACKEND_SERVICE_IP=${HOST_IP_EXTERNAL}
27+
export CHATQNA_BACKEND_SERVICE_PORT=18102
28+
export CHATQNA_INDEX_NAME="rag-redis"
29+
export CHATQNA_MEGA_SERVICE_HOST_IP=${HOST_IP}
30+
export CHATQNA_RETRIEVER_SERVICE_HOST_IP=${HOST_IP}
31+
export CHATQNA_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna"
32+
export CHATQNA_DATAPREP_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep"
33+
export CHATQNA_DATAPREP_GET_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/get_file"
34+
export CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/delete_file"
35+
export CHATQNA_REDIS_URL="redis://${HOST_IP}:${CHATQNA_REDIS_VECTOR_PORT}"
36+
export CHATQNA_EMBEDDING_SERVICE_HOST_IP=${HOST_IP}
37+
export CHATQNA_RERANK_SERVICE_HOST_IP=${HOST_IP}
38+
export CHATQNA_LLM_SERVICE_HOST_IP=${HOST_IP}
39+
export CHATQNA_NGINX_PORT=18104

0 commit comments

Comments
 (0)