From e433f75e57b06c6d14a1a68b9aad40aabbb56ff8 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Fri, 24 Jan 2025 20:29:42 +0700 Subject: [PATCH 001/180] ChatQnA - add deploy app with vLLM ROCm Signed-off-by: Chingis Yundunov --- ChatQnA/Dockerfile-vllm-rocm | 18 + .../amd/gpu/rocm/README_vLLM.md | 439 ++++++++++++++++++ .../amd/gpu/rocm/compose_vllm.yaml | 169 +++++++ .../amd/gpu/rocm/set_env_vllm.sh | 39 ++ ChatQnA/docker_image_build/build.yaml | 9 + ChatQnA/tests/test_compose_on_rocm_vllm.sh | 265 +++++++++++ 6 files changed, 939 insertions(+) create mode 100644 ChatQnA/Dockerfile-vllm-rocm create mode 100644 ChatQnA/docker_compose/amd/gpu/rocm/README_vLLM.md create mode 100644 ChatQnA/docker_compose/amd/gpu/rocm/compose_vllm.yaml create mode 100644 ChatQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh create mode 100644 ChatQnA/tests/test_compose_on_rocm_vllm.sh diff --git a/ChatQnA/Dockerfile-vllm-rocm b/ChatQnA/Dockerfile-vllm-rocm new file mode 100644 index 0000000000..0b547db9d8 --- /dev/null +++ b/ChatQnA/Dockerfile-vllm-rocm @@ -0,0 +1,18 @@ +FROM rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 + +# Set the working directory +WORKDIR /workspace + +# Copy the api_server.py into the image +ADD https://raw.githubusercontent.com/ROCm/vllm/a466f09d7f20ca073f21e3f64b8c9487e4c4ff4b/vllm/entrypoints/sync_openai/api_server.py /workspace/api_server.py + +# Expose the port used by the API server +EXPOSE 8011 + +# Set environment variables +ENV HUGGINGFACE_HUB_CACHE=/workspace +ENV WILM_USE_TRITON_FLASH_ATTENTION=0 +ENV PYTORCH_JIT=0 + +# Set the entrypoint to the api_server.py script +ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README_vLLM.md b/ChatQnA/docker_compose/amd/gpu/rocm/README_vLLM.md new file mode 100644 index 0000000000..66d0c9b01b --- /dev/null +++ b/ChatQnA/docker_compose/amd/gpu/rocm/README_vLLM.md @@ -0,0 +1,439 @@ +# Build and deploy CodeGen Application on AMD GPU (ROCm) + +## Build MegaService of ChatQnA on AMD ROCm GPU + +This document outlines the deployment process for a ChatQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on AMD ROCm GPU platform. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as embedding, retriever, rerank, and llm. We will publish the Docker images to Docker Hub, it will simplify the deployment process for this service. + +Quick Start Deployment Steps: + +1. Set up the environment variables. +2. Run Docker Compose. +3. Consume the ChatQnA Service. + +## Quick Start: 1.Setup Environment Variable + +To set up environment variables for deploying ChatQnA services, follow these steps: + +1. Set the required environment variables: + + ```bash + # Example: host_ip="192.168.1.1" + export HOST_IP=${host_ip} + # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1" + export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} + ``` + +2. If you are in a proxy environment, also set the proxy-related environment variables: + + ```bash + export http_proxy="Your_HTTP_Proxy" + export https_proxy="Your_HTTPs_Proxy" + ``` + +3. Set up other environment variables: + + ```bash + source ./set_env.sh + ``` + +## Quick Start: 2.Run Docker Compose + +```bash +docker compose up -d +``` + +It will automatically download the docker image on `docker hub`: + +```bash +docker pull opea/chatqna:latest +docker pull opea/chatqna-ui:latest +``` + +In following cases, you could build docker image from source by yourself. + +- Failed to download the docker image. + +- If you want to use a specific version of Docker image. + +Please refer to 'Build Docker Images' in below. + +## QuickStart: 3.Consume the ChatQnA Service + +Prepare and upload test document + +``` +# download pdf file +wget https://raw.githubusercontent.com/opea-project/GenAIComps/main/comps/retrievers/redis/data/nke-10k-2023.pdf +# upload pdf file with dataprep +curl -X POST "http://${host_ip}:6007/v1/dataprep" \ + -H "Content-Type: multipart/form-data" \ + -F "files=@./nke-10k-2023.pdf" +``` + +Get MegaSerice(backend) response: + +```bash +curl http://${host_ip}:8888/v1/chatqna \ + -H "Content-Type: application/json" \ + -d '{ + "messages": "What is the revenue of Nike in 2023?" + }' +``` + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally. This step can be ignored after the Docker images published to Docker hub. + +### 1. Source Code install GenAIComps + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +``` + +### 2. Build Retriever Image + +```bash +docker build --no-cache -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile . +``` + +### 3. Build Dataprep Image + +```bash +docker build --no-cache -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/langchain/Dockerfile . +``` + +### 4. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `chatqna.py` Python script. Build the MegaService Docker image using the command below: + +```bash +git clone https://github.com/opea-project/GenAIExamples.git +cd GenAIExamples/ChatQnA/docker +docker build --no-cache -t opea/chatqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +cd ../../.. +``` + +### 5. Build UI Docker Image + +Construct the frontend Docker image using the command below: + +```bash +cd GenAIExamples/ChatQnA/ui +docker build --no-cache -t opea/chatqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +cd ../../../.. +``` + +### 6. Build React UI Docker Image (Optional) + +Construct the frontend Docker image using the command below: + +```bash +cd GenAIExamples/ChatQnA/ui +docker build --no-cache -t opea/chatqna-react-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +cd ../../../.. +``` + +### 7. Build Nginx Docker Image + +```bash +cd GenAIComps +docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile . +``` + +### 8. Build vLLM-ROCm Docker Image + +```bash +cd GenAIExamples/ChatQnA +docker build -t opea/llm-vllm-rocm:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker_compose/amd/gpu/rocm-vllm/Dockerfile-vllm . +``` + +Then run the command `docker images`, you will have the following 5 Docker Images: + +1. `opea/retriever-redis:latest` +2. `opea/dataprep-redis:latest` +3. `opea/chatqna:latest` +4. `opea/chatqna-ui:latest` or `opea/chatqna-react-ui:latest` +5. `opea/nginx:latest` + +## 🚀 Start MicroServices and MegaService + +### Required Models + +By default, the embedding, reranking and LLM models are set to a default value as listed below: + +| Service | Model | +| --------- | ------------------------- | +| Embedding | BAAI/bge-base-en-v1.5 | +| Reranking | BAAI/bge-reranker-base | +| LLM | Intel/neural-chat-7b-v3-3 | + +Change the `xxx_MODEL_ID` below for your needs. + +### Setup Environment Variables + +1. Set the required environment variables: + + ```bash + # Example: host_ip="192.168.1.1" + export host_ip="External_Public_IP" + # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1" + export no_proxy="Your_No_Proxy" + export CHATQNA_HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token" + # Example: NGINX_PORT=80 + export HOST_IP=${host_ip} + export NGINX_PORT=${your_nginx_port} + export CHATQNA_TGI_SERVICE_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" + export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" + export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base" + export CHATQNA_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" + export CHATQNA_TGI_SERVICE_PORT=8008 + export CHATQNA_TEI_EMBEDDING_PORT=8090 + export CHATQNA_TEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}" + export CHATQNA_TEI_RERANKING_PORT=8808 + export CHATQNA_REDIS_VECTOR_PORT=16379 + export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001 + export CHATQNA_REDIS_DATAPREP_PORT=6007 + export CHATQNA_REDIS_RETRIEVER_PORT=7000 + export CHATQNA_INDEX_NAME="rag-redis" + export CHATQNA_MEGA_SERVICE_HOST_IP=${HOST_IP} + export CHATQNA_RETRIEVER_SERVICE_HOST_IP=${HOST_IP} + export CHATQNA_BACKEND_SERVICE_ENDPOINT="http://127.0.0.1:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna" + export CHATQNA_DATAPREP_SERVICE_ENDPOINT="http://127.0.0.1:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep" + export CHATQNA_DATAPREP_GET_FILE_ENDPOINT="http://127.0.0.1:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/get_file" + export CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT="http://127.0.0.1:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/delete_file" + export CHATQNA_FRONTEND_SERVICE_IP=${HOST_IP} + export CHATQNA_FRONTEND_SERVICE_PORT=5173 + export CHATQNA_BACKEND_SERVICE_NAME=chatqna + export CHATQNA_BACKEND_SERVICE_IP=${HOST_IP} + export CHATQNA_BACKEND_SERVICE_PORT=8888 + export CHATQNA_REDIS_URL="redis://${HOST_IP}:${CHATQNA_REDIS_VECTOR_PORT}" + export CHATQNA_EMBEDDING_SERVICE_HOST_IP=${HOST_IP} + export CHATQNA_RERANK_SERVICE_HOST_IP=${HOST_IP} + export CHATQNA_LLM_SERVICE_HOST_IP=${HOST_IP} + export CHATQNA_NGINX_PORT=5176 + ``` + +2. If you are in a proxy environment, also set the proxy-related environment variables: + + ```bash + export http_proxy="Your_HTTP_Proxy" + export https_proxy="Your_HTTPs_Proxy" + ``` + +3. Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) into tgi-service in compose.yaml file + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +4. Set up other environment variables: + + ```bash + source ./set_env.sh + ``` + +### Start all the services Docker Containers + +```bash +cd GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm +docker compose up -d +``` + +### Validate MicroServices and MegaService + +1. TEI Embedding Service + + ```bash + curl ${host_ip}:8090/embed \ + -X POST \ + -d '{"inputs":"What is Deep Learning?"}' \ + -H 'Content-Type: application/json' + ``` + +2. Retriever Microservice + + To consume the retriever microservice, you need to generate a mock embedding vector by Python script. The length of embedding vector + is determined by the embedding model. + Here we use the model `EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"`, which vector size is 768. + + Check the vecotor dimension of your embedding model, set `your_embedding` dimension equals to it. + + ```bash + export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + curl http://${host_ip}:7000/v1/retrieval \ + -X POST \ + -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \ + -H 'Content-Type: application/json' + ``` + +3. TEI Reranking Service + + ```bash + curl http://${host_ip}:8808/rerank \ + -X POST \ + -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \ + -H 'Content-Type: application/json' + ``` + +4. vLLM Service + + In first startup, this service will take more time to download the model files. After it's finished, the service will be ready. + + Try the command below to check whether the vLLM service is ready. + + ```bash + docker logs ${CONTAINER_ID} | grep "Application startup complete" + ``` + + If the service is ready, you will get the response like below. + + ``` + 2024-09-03T02:47:53.402023Z INFO text_generation_router::server: router/src/server.rs:2311: Connected + ``` + + Then try the `cURL` command below to validate TGI. + + ```bash + curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' + ``` + +5. MegaService + + ```bash + curl http://${host_ip}:8888/v1/chatqna -H "Content-Type: application/json" -d '{ + "messages": "What is the revenue of Nike in 2023?" + }' + ``` + +6. Nginx Service + + ```bash + curl http://${host_ip}:${NGINX_PORT}/v1/chatqna \ + -H "Content-Type: application/json" \ + -d '{"messages": "What is the revenue of Nike in 2023?"}' + ``` + +7. Dataprep Microservice（Optional） + +If you want to update the default knowledge base, you can use the following commands: + +Update Knowledge Base via Local File Upload: + +```bash +curl -X POST "http://${host_ip}:6007/v1/dataprep" \ + -H "Content-Type: multipart/form-data" \ + -F "files=@./nke-10k-2023.pdf" +``` + +This command updates a knowledge base by uploading a local file for processing. Update the file path according to your environment. + +Add Knowledge Base via HTTP Links: + +```bash +curl -X POST "http://${host_ip}:6007/v1/dataprep" \ + -H "Content-Type: multipart/form-data" \ + -F 'link_list=["https://opea.dev"]' +``` + +This command updates a knowledge base by submitting a list of HTTP links for processing. + +Also, you are able to get the file list that you uploaded: + +```bash +curl -X POST "http://${host_ip}:6007/v1/dataprep/get_file" \ + -H "Content-Type: application/json" +``` + +To delete the file/link you uploaded: + +```bash +# delete link +curl -X POST "http://${host_ip}:6007/v1/dataprep/delete_file" \ + -d '{"file_path": "https://opea.dev"}' \ + -H "Content-Type: application/json" + +# delete file +curl -X POST "http://${host_ip}:6007/v1/dataprep/delete_file" \ + -d '{"file_path": "nke-10k-2023.pdf"}' \ + -H "Content-Type: application/json" + +# delete all uploaded files and links +curl -X POST "http://${host_ip}:6007/v1/dataprep/delete_file" \ + -d '{"file_path": "all"}' \ + -H "Content-Type: application/json" +``` + +## 🚀 Launch the UI + +### Launch with origin port + +To access the frontend, open the following URL in your browser: http://{host_ip}:5173. By default, the UI runs on port 5173 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `compose.yaml` file as shown below: + +```yaml + chaqna-ui-server: + image: opea/chatqna-ui:latest + ... + ports: + - "80:5173" +``` + +### Launch with Nginx + +If you want to launch the UI using Nginx, open this URL: `http://${host_ip}:${NGINX_PORT}` in your browser to access the frontend. + +## 🚀 Launch the Conversational UI (Optional) + +To access the Conversational UI (react based) frontend, modify the UI service in the `compose.yaml` file. Replace `chaqna-ui-server` service with the `chatqna-react-ui-server` service as per the config below: + +```yaml +chatqna-react-ui-server: + image: opea/chatqna-react-ui:latest + container_name: chatqna-react-ui-server + environment: + - APP_BACKEND_SERVICE_ENDPOINT=${BACKEND_SERVICE_ENDPOINT} + - APP_DATA_PREP_SERVICE_URL=${DATAPREP_SERVICE_ENDPOINT} + ports: + - "5174:80" + depends_on: + - chaqna-backend-server + ipc: host + restart: always +``` + +Once the services are up, open the following URL in your browser: http://{host_ip}:5174. By default, the UI runs on port 80 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `compose.yaml` file as shown below: + +```yaml + chaqna-react-ui-server: + image: opea/chatqna-react-ui:latest + ... + ports: + - "80:80" +``` + +![project-screenshot](../../../../assets/img/chat_ui_init.png) + +Here is an example of running ChatQnA: + +![project-screenshot](../../../../assets/img/chat_ui_response.png) + +Here is an example of running ChatQnA with Conversational UI (React): + +![project-screenshot](../../../../assets/img/conversation_ui_response.png) diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/compose_vllm.yaml b/ChatQnA/docker_compose/amd/gpu/rocm/compose_vllm.yaml new file mode 100644 index 0000000000..0b956d3a2a --- /dev/null +++ b/ChatQnA/docker_compose/amd/gpu/rocm/compose_vllm.yaml @@ -0,0 +1,169 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + chatqna-redis-vector-db: + image: redis/redis-stack:7.2.0-v9 + container_name: chatqna-redis-vector-db + ports: + - "${CHATQNA_REDIS_VECTOR_PORT:-6379}:6379" + - "${CHATQNA_REDIS_VECTOR_INSIGHT_PORT:-8001}:8001" + chatqna-dataprep-service: + image: ${REGISTRY:-opea}/dataprep:${TAG:-latest} + container_name: chatqna-dataprep-service + depends_on: + - chatqna-redis-vector-db + - chatqna-tei-embedding-service + ports: + - "${CHATQNA_REDIS_DATAPREP_PORT}:6007" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + REDIS_URL: ${CHATQNA_REDIS_URL} + INDEX_NAME: ${CHATQNA_INDEX_NAME} + TEI_ENDPOINT: ${CHATQNA_TEI_EMBEDDING_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${CHATQNA_HUGGINGFACEHUB_API_TOKEN} + chatqna-tei-embedding-service: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 + container_name: chatqna-tei-embedding-service + ports: + - "${CHATQNA_TEI_EMBEDDING_PORT}:80" + volumes: + - "./data:/data" + shm_size: 1g + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + command: --model-id ${CHATQNA_EMBEDDING_MODEL_ID} --auto-truncate + chatqna-retriever: + image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} + container_name: chatqna-retriever + depends_on: + - chatqna-redis-vector-db + ports: + - "${CHATQNA_REDIS_RETRIEVER_PORT}:7000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + REDIS_URL: ${CHATQNA_REDIS_URL} + INDEX_NAME: ${CHATQNA_INDEX_NAME} + TEI_EMBEDDING_ENDPOINT: ${CHATQNA_TEI_EMBEDDING_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${CHATQNA_HUGGINGFACEHUB_API_TOKEN} + restart: unless-stopped + chatqna-tei-reranking-service: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 + container_name: chatqna-tei-reranking-service + ports: + - "${CHATQNA_TEI_RERANKING_PORT}:80" + volumes: + - "./data:/data" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${CHATQNA_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + command: --model-id ${CHATQNA_RERANK_MODEL_ID} --auto-truncate + chatqna-vllm-service: + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} + container_name: chatqna-vllm-service + ports: + - "${CHATQNA_VLLM_SERVICE_PORT:-8081}:8011" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${CHATQNA_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${CHATQNA_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + WILM_USE_TRITON_FLASH_ATTENTION: 0 + PYTORCH_JIT: 0 + volumes: + - "./data:/data" + shm_size: 128G + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + - apparmor=unconfined + command: "--model ${CHATQNA_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" + ipc: host + chatqna-backend-server: + image: ${REGISTRY:-opea}/chatqna:${TAG:-latest} + container_name: chatqna-backend-server + depends_on: + - chatqna-redis-vector-db + - chatqna-tei-embedding-service + - chatqna-retriever + - chatqna-tei-reranking-service + - chatqna-vllm-service + ports: + - "${CHATQNA_BACKEND_SERVICE_PORT}:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${CHATQNA_MEGA_SERVICE_HOST_IP} + EMBEDDING_SERVER_HOST_IP: ${HOST_IP} + EMBEDDING_SERVER_PORT: ${CHATQNA_TEI_EMBEDDING_PORT:-80} + RETRIEVER_SERVICE_HOST_IP: ${HOST_IP} + RERANK_SERVER_HOST_IP: ${HOST_IP} + RERANK_SERVER_PORT: ${CHATQNA_TEI_RERANKING_PORT:-80} + LLM_SERVER_HOST_IP: ${HOST_IP} + LLM_SERVER_PORT: ${CHATQNA_VLLM_SERVICE_PORT:-80} + LLM_MODEL: ${CHATQNA_LLM_MODEL_ID} + ipc: host + restart: always + chatqna-ui-server: + image: ${REGISTRY:-opea}/chatqna-ui:${TAG:-latest} + container_name: chatqna-ui-server + depends_on: + - chatqna-backend-server + ports: + - "${CHATQNA_FRONTEND_SERVICE_PORT}:5173" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + CHAT_BASE_URL: ${CHATQNA_BACKEND_SERVICE_ENDPOINT} + UPLOAD_FILE_BASE_URL: ${CHATQNA_DATAPREP_SERVICE_ENDPOINT} + GET_FILE: ${CHATQNA_DATAPREP_GET_FILE_ENDPOINT} + DELETE_FILE: ${CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT} + ipc: host + restart: always + chatqna-nginx-server: + image: ${REGISTRY:-opea}/nginx:${TAG:-latest} + container_name: chatqna-nginx-server + depends_on: + - chatqna-backend-server + - chatqna-ui-server + ports: + - "${CHATQNA_NGINX_PORT}:80" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + FRONTEND_SERVICE_IP: ${CHATQNA_FRONTEND_SERVICE_IP} + FRONTEND_SERVICE_PORT: ${CHATQNA_FRONTEND_SERVICE_PORT} + BACKEND_SERVICE_NAME: ${CHATQNA_BACKEND_SERVICE_NAME} + BACKEND_SERVICE_IP: ${CHATQNA_BACKEND_SERVICE_IP} + BACKEND_SERVICE_PORT: ${CHATQNA_BACKEND_SERVICE_PORT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh b/ChatQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh new file mode 100644 index 0000000000..e8a56ef9be --- /dev/null +++ b/ChatQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP='10.53.22.29' +export HOST_IP_EXTERNAL='68.69.180.77' +export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" +export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base" +export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" +export MODEL=${CHATQNA_LLM_MODEL_ID} +export CHATQNA_VLLM_SERVICE_PORT=18008 +export CHATQNA_TEI_EMBEDDING_PORT=18090 +export CHATQNA_TEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}" +export CHATQNA_TEI_RERANKING_PORT=18808 +export CHATQNA_REDIS_VECTOR_PORT=16379 +export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001 +export CHATQNA_REDIS_DATAPREP_PORT=18103 +export CHATQNA_REDIS_RETRIEVER_PORT=7000 +export CHATQNA_FRONTEND_SERVICE_IP=${HOST_IP} +export CHATQNA_FRONTEND_SERVICE_PORT=18101 +export CHATQNA_BACKEND_SERVICE_NAME=chatqna +export CHATQNA_BACKEND_SERVICE_IP=${HOST_IP_EXTERNAL} +export CHATQNA_BACKEND_SERVICE_PORT=18102 +export CHATQNA_INDEX_NAME="rag-redis" +export CHATQNA_MEGA_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_RETRIEVER_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna" +export CHATQNA_DATAPREP_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep" +export CHATQNA_DATAPREP_GET_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/get_file" +export CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/delete_file" +export CHATQNA_REDIS_URL="redis://${HOST_IP}:${CHATQNA_REDIS_VECTOR_PORT}" +export CHATQNA_EMBEDDING_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_RERANK_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_LLM_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_NGINX_PORT=18104 diff --git a/ChatQnA/docker_image_build/build.yaml b/ChatQnA/docker_image_build/build.yaml index 33cd2b607f..df5636e0ce 100644 --- a/ChatQnA/docker_image_build/build.yaml +++ b/ChatQnA/docker_image_build/build.yaml @@ -71,6 +71,15 @@ services: dockerfile: comps/guardrails/src/guardrails/Dockerfile extends: chatqna image: ${REGISTRY:-opea}/guardrails:${TAG:-latest} + vllm_rocm: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: ../ + dockerfile: ./Dockerfile-vllm-rocm + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} vllm: build: context: vllm diff --git a/ChatQnA/tests/test_compose_on_rocm_vllm.sh b/ChatQnA/tests/test_compose_on_rocm_vllm.sh new file mode 100644 index 0000000000..df11236367 --- /dev/null +++ b/ChatQnA/tests/test_compose_on_rocm_vllm.sh @@ -0,0 +1,265 @@ +#!/bin/bash +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +export HOST_IP=${ip_address} +export HOST_IP_EXTERNAL=${ip_address} +export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" +export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base" +export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" +export CHATQNA_VLLM_SERVICE_PORT=9009 +export CHATQNA_TEI_EMBEDDING_PORT=8090 +export CHATQNA_TEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}" +export CHATQNA_TEI_RERANKING_PORT=8808 +export CHATQNA_REDIS_VECTOR_PORT=6379 +export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001 +export CHATQNA_REDIS_DATAPREP_PORT=6007 +export CHATQNA_REDIS_RETRIEVER_PORT=7000 +export CHATQNA_INDEX_NAME="rag-redis" +export CHATQNA_MEGA_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_RETRIEVER_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_FRONTEND_SERVICE_IP=${HOST_IP} +export CHATQNA_FRONTEND_SERVICE_PORT=5173 +export CHATQNA_BACKEND_SERVICE_NAME=chatqna +export CHATQNA_BACKEND_SERVICE_IP=${HOST_IP} +export CHATQNA_BACKEND_SERVICE_PORT=8888 +export CHATQNA_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna" +export CHATQNA_DATAPREP_SERVICE_ENDPOINT="http://${HOST_IP}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep" +export CHATQNA_DATAPREP_GET_FILE_ENDPOINT="http://${HOST_IP}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/get_file" +export CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT="http://${HOST_IP}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/delete_file" +export CHATQNA_REDIS_URL="redis://${HOST_IP}:${CHATQNA_REDIS_VECTOR_PORT}" +export CHATQNA_EMBEDDING_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_RERANK_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_LLM_SERVICE_HOST_IP=${HOST_IP} +export CHATQNA_NGINX_PORT=8081 +export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export PATH="/home/huggingface/miniconda3/bin:$PATH" + +function build_docker_images() { + cd "$WORKPATH"/docker_image_build + git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="llm-vllm-rocm chatqna chatqna-ui dataprep retriever nginx" + docker compose -f build.yaml build ${service_list} --no-cache > "${LOG_PATH}"/docker_image_build.log + +# docker pull vllm-api-server +# docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 + + docker images && sleep 1s +} + +function start_services() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + + # Start Docker Containers + docker compose -f compose_vllm.yaml up -d > "${LOG_PATH}"/start_services_with_compose.log + + n=0 + until [[ "$n" -ge 500 ]]; do + docker logs chatqna-vllm-service >& "${LOG_PATH}"/chatqna-vllm-service_start.log + if grep -q "Application startup complete" "${LOG_PATH}"/chatqna-vllm-service_start.log; then + break + fi + sleep 20s + n=$((n+1)) + done +} + +function validate_service() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + if [[ $SERVICE_NAME == *"dataprep_upload_file"* ]]; then + cd "$LOG_PATH" + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -F 'files=@./dataprep_file.txt' -H 'Content-Type: multipart/form-data' "$URL") + elif [[ $SERVICE_NAME == *"dataprep_upload_link"* ]]; then + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -F 'link_list=["https://www.ces.tech/"]' "$URL") + elif [[ $SERVICE_NAME == *"dataprep_get"* ]]; then + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -H 'Content-Type: application/json' "$URL") + elif [[ $SERVICE_NAME == *"dataprep_del"* ]]; then + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -d '{"file_path": "all"}' -H 'Content-Type: application/json' "$URL") + else + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + fi + HTTP_STATUS=$(echo "$HTTP_RESPONSE" | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + RESPONSE_BODY=$(echo "$HTTP_RESPONSE" | sed -e 's/HTTPSTATUS\:.*//g') + + docker logs "${DOCKER_NAME}" >> "${LOG_PATH}"/"${SERVICE_NAME}".log + + # check response status + if [ "$HTTP_STATUS" -ne "200" ]; then + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + exit 1 + else + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + fi + # check response body + if [[ "$RESPONSE_BODY" != *"$EXPECTED_RESULT"* ]]; then + echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY" + exit 1 + else + echo "[ $SERVICE_NAME ] Content is as expected." + fi + + sleep 1s +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # tei for embedding service + validate_service \ + "${ip_address}:8090/embed" \ + "[[" \ + "chatqna-tei-embedding-service" \ + "chatqna-tei-embedding-service" \ + '{"inputs":"What is Deep Learning?"}' + + sleep 1m # retrieval can't curl as expected, try to wait for more time + + # test /v1/dataprep upload file + echo "Deep learning is a subset of machine learning that utilizes neural networks with multiple layers to analyze various levels of abstract data representations. It enables computers to identify patterns and make decisions with minimal human intervention by learning from large amounts of data." > "$LOG_PATH"/dataprep_file.txt + validate_service \ + "http://${ip_address}:6007/v1/dataprep" \ + "Data preparation succeeded" \ + "dataprep_upload_file" \ + "chatqna-dataprep-redis-service" + + # test /v1/dataprep upload link + validate_service \ + "http://${ip_address}:6007/v1/dataprep" \ + "Data preparation succeeded" \ + "dataprep_upload_link" \ + "chatqna-dataprep-redis-service" + + # test /v1/dataprep/get_file + validate_service \ + "http://${ip_address}:6007/v1/dataprep/get_file" \ + '{"name":' \ + "dataprep_get" \ + "chatqna-dataprep-redis-service" + + # test /v1/dataprep/delete_file + validate_service \ + "http://${ip_address}:6007/v1/dataprep/delete_file" \ + '{"status":true}' \ + "dataprep_del" \ + "chatqna-dataprep-redis-service" + + # retrieval microservice + test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + validate_service \ + "${ip_address}:7000/v1/retrieval" \ + "retrieved_docs" \ + "chatqna-retriever" \ + "chatqna-retriever" \ + "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" + + # tei for rerank microservice + validate_service \ + "${ip_address}:8808/rerank" \ + '{"index":1,"score":' \ + "chatqna-tei-reranking-service" \ + "chatqna-tei-reranking-service" \ + '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' + + # tgi for llm service + validate_service \ + "${ip_address}:9009/v1/chat/completions" \ + "\"content\":\"" \ + "chatqna-vllm-service" \ + "chatqna-vllm-service" \ + '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' + +} + +function validate_megaservice() { + # Curl the Mega Service + validate_service \ + "${ip_address}:8888/v1/chatqna" \ + "data: " \ + "chatqna-backend-server" \ + "chatqna-backend-server" \ + '{"messages": "What is the revenue of Nike in 2023?"}' + +} + +function validate_frontend() { + echo "[ TEST INFO ]: --------- frontend test started ---------" + cd "$WORKPATH"/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + if conda info --envs | grep -q "$conda_env_name"; then + echo "$conda_env_name exist!" + else + conda create -n ${conda_env_name} python=3.12 -y + fi + source activate ${conda_env_name} + echo "[ TEST INFO ]: --------- conda env activated ---------" + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs=22.6.0 -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + docker compose -f compose_vllm.yaml stop && docker compose -f compose_vllm.yaml rm -f +} + +function main() { + + stop_docker +# if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_time=$(date +%s) + start_services + end_time=$(date +%s) + duration=$((end_time-start_time)) + echo "Mega service start duration is $duration s" && sleep 1s + + + if [ "${mode}" == "perf" ]; then + python3 "$WORKPATH"/tests/chatqna_benchmark.py + elif [ "${mode}" == "" ]; then + validate_microservices + echo "==== microservices validated ====" + validate_megaservice + echo "==== megaservice validated ====" + validate_frontend + echo "==== frontend validated ====" + fi + + stop_docker + echo y | docker system prune + +} + +main From 5a15be3c8969ca060df9fe34e9874ff4c705c06a Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Fri, 24 Jan 2025 20:33:56 +0700 Subject: [PATCH 002/180] ChatQnA - fix deploy app with vLLM ROCm Signed-off-by: Chingis Yundunov --- ChatQnA/tests/test_compose_on_rocm_vllm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChatQnA/tests/test_compose_on_rocm_vllm.sh b/ChatQnA/tests/test_compose_on_rocm_vllm.sh index df11236367..8f3832814f 100644 --- a/ChatQnA/tests/test_compose_on_rocm_vllm.sh +++ b/ChatQnA/tests/test_compose_on_rocm_vllm.sh @@ -238,7 +238,7 @@ function stop_docker() { function main() { stop_docker -# if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi start_time=$(date +%s) start_services end_time=$(date +%s) From 43086d6a2cea74d637170be9edf0d61a29e7342f Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Fri, 24 Jan 2025 20:34:39 +0700 Subject: [PATCH 003/180] ChatQnA - fix deploy app with vLLM ROCm Signed-off-by: Chingis Yundunov --- ChatQnA/tests/test_compose_on_rocm_vllm.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ChatQnA/tests/test_compose_on_rocm_vllm.sh b/ChatQnA/tests/test_compose_on_rocm_vllm.sh index 8f3832814f..99ce721b68 100644 --- a/ChatQnA/tests/test_compose_on_rocm_vllm.sh +++ b/ChatQnA/tests/test_compose_on_rocm_vllm.sh @@ -62,7 +62,7 @@ function build_docker_images() { } function start_services() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + cd "$WORKPATH"/docker_compose/amd/gpu/rocm # Start Docker Containers docker compose -f compose_vllm.yaml up -d > "${LOG_PATH}"/start_services_with_compose.log @@ -231,7 +231,7 @@ function validate_frontend() { } function stop_docker() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + cd "$WORKPATH"/docker_compose/amd/gpu/rocm docker compose -f compose_vllm.yaml stop && docker compose -f compose_vllm.yaml rm -f } From 5b096afcaac5b1ccd5e42462a00a56b3ddef44e0 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Fri, 24 Jan 2025 20:35:06 +0700 Subject: [PATCH 004/180] ChatQnA - fix deploy app with vLLM ROCm Signed-off-by: Chingis Yundunov --- ChatQnA/tests/test_compose_on_rocm_vllm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChatQnA/tests/test_compose_on_rocm_vllm.sh b/ChatQnA/tests/test_compose_on_rocm_vllm.sh index 99ce721b68..11a47444ab 100644 --- a/ChatQnA/tests/test_compose_on_rocm_vllm.sh +++ b/ChatQnA/tests/test_compose_on_rocm_vllm.sh @@ -52,7 +52,7 @@ function build_docker_images() { git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="llm-vllm-rocm chatqna chatqna-ui dataprep retriever nginx" + service_list="vllm_rocm chatqna chatqna-ui dataprep retriever nginx" docker compose -f build.yaml build ${service_list} --no-cache > "${LOG_PATH}"/docker_image_build.log # docker pull vllm-api-server From 8d93803bf17e7ad90d3fa8b9f285e79759eb98dc Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Fri, 24 Jan 2025 20:51:09 +0700 Subject: [PATCH 005/180] ChatQnA - fix deploy app with vLLM ROCm Signed-off-by: Chingis Yundunov --- ChatQnA/ui/svelte/.env | 6 +- ChatQnA/ui/svelte/.gitignore | 5 + ChatQnA/ui/svelte/.npmrc | 1 + ChatQnA/ui/svelte/README.md | 27 +- ChatQnA/ui/svelte/package.json | 48 +- ChatQnA/ui/svelte/playwright.config.ts | 109 ++-- ChatQnA/ui/svelte/postcss.config.cjs | 12 +- ChatQnA/ui/svelte/src/app.d.ts | 16 +- ChatQnA/ui/svelte/src/app.html | 18 +- ChatQnA/ui/svelte/src/app.pcss | 4 + .../src/lib/assets/loadingAnimation.svelte | 48 ++ .../src/lib/assets/translateIcon.svelte | 19 + ChatQnA/ui/svelte/src/lib/header.svelte | 32 ++ ChatQnA/ui/svelte/src/lib/shared/Network.ts | 32 ++ ChatQnA/ui/svelte/src/lib/shared/constant.ts | 38 ++ ChatQnA/ui/svelte/src/routes/+layout.svelte | 29 +- ChatQnA/ui/svelte/src/routes/+page.svelte | 518 ++++++++---------- ChatQnA/ui/svelte/src/routes/types.d.ts | 21 + ChatQnA/ui/svelte/svelte.config.js | 29 +- ChatQnA/ui/svelte/tailwind.config.cjs | 44 +- ChatQnA/ui/svelte/tests/codeTrans.spec.ts | 28 + ChatQnA/ui/svelte/tsconfig.json | 27 +- ChatQnA/ui/svelte/vite.config.ts | 16 +- 23 files changed, 607 insertions(+), 520 deletions(-) create mode 100644 ChatQnA/ui/svelte/.gitignore create mode 100644 ChatQnA/ui/svelte/.npmrc create mode 100644 ChatQnA/ui/svelte/src/app.pcss create mode 100644 ChatQnA/ui/svelte/src/lib/assets/loadingAnimation.svelte create mode 100644 ChatQnA/ui/svelte/src/lib/assets/translateIcon.svelte create mode 100644 ChatQnA/ui/svelte/src/lib/header.svelte create mode 100644 ChatQnA/ui/svelte/src/lib/shared/Network.ts create mode 100644 ChatQnA/ui/svelte/src/lib/shared/constant.ts create mode 100644 ChatQnA/ui/svelte/src/routes/types.d.ts create mode 100644 ChatQnA/ui/svelte/tests/codeTrans.spec.ts diff --git a/ChatQnA/ui/svelte/.env b/ChatQnA/ui/svelte/.env index 28aeea4f7b..42d6c60cfe 100644 --- a/ChatQnA/ui/svelte/.env +++ b/ChatQnA/ui/svelte/.env @@ -1,7 +1,3 @@ -CHAT_BASE_URL = '/v1/chatqna' +BASE_URL = '/v1/codetrans' -UPLOAD_FILE_BASE_URL = '/v1/dataprep/ingest' -GET_FILE = '/v1/dataprep/get' - -DELETE_FILE = '/v1/dataprep/delete' diff --git a/ChatQnA/ui/svelte/.gitignore b/ChatQnA/ui/svelte/.gitignore new file mode 100644 index 0000000000..285c881e35 --- /dev/null +++ b/ChatQnA/ui/svelte/.gitignore @@ -0,0 +1,5 @@ +*/node_modules +/build +/dist +*/.svelte-kit + diff --git a/ChatQnA/ui/svelte/.npmrc b/ChatQnA/ui/svelte/.npmrc new file mode 100644 index 0000000000..b6f27f1359 --- /dev/null +++ b/ChatQnA/ui/svelte/.npmrc @@ -0,0 +1 @@ +engine-strict=true diff --git a/ChatQnA/ui/svelte/README.md b/ChatQnA/ui/svelte/README.md index d3c26b8f0f..a224f08806 100644 --- a/ChatQnA/ui/svelte/README.md +++ b/ChatQnA/ui/svelte/README.md @@ -1,22 +1,16 @@ -# ChatQnA Customized UI +# Code Translation ## 📸 Project Screenshots -![project-screenshot](../../assets/img/chat_ui_init.png) -![project-screenshot](../../assets/img/chat_ui_response.png) -![project-screenshot](../../assets/img/chat_ui_upload.png) +![project-screenshot](../../assets/img/codeTrans_ui_init.png) +![project-screenshot](../../assets/img/codeTrans_ui_select.png) +![project-screenshot](../../assets/img/codeTrans_ui_response.png) ## 🧐 Features Here're some of the project's features: -- Start a Text Chat：Initiate a text chat with the ability to input written conversations, where the dialogue content can also be customized based on uploaded files. -- Clear: Clear the record of the current dialog box without retaining the contents of the dialog box. -- Chat history: Historical chat records can still be retained after refreshing, making it easier for users to view the context. -- Scroll to Bottom / Top: The chat automatically slides to the bottom. Users can also click the top icon to slide to the top of the chat record. -- End to End Time: Shows the time spent on the current conversation. -- Upload File: The choice between uploading locally or copying a remote link. Chat according to uploaded knowledge base. -- Delete File: Delete a certain uploaded file. +- Code Translation: The system is capable of recognizing multiple languages and converting the current code content into the desired language's code format, enabling a set of codes to be reused in multiple places, thus alleviating developers' development pressure. ## 🛠️ Get it Running @@ -25,18 +19,9 @@ Here're some of the project's features: 2. cd command to the current folder. 3. Modify the required .env variables. - ``` - CHAT_BASE_URL = '' - - UPLOAD_FILE_BASE_URL = '' - - GET_FILE = '' - - DELETE_FILE = '' - + BASE_URL = '' ``` - 4. Execute `npm install` to install the corresponding dependencies. 5. Execute `npm run dev` in both environments diff --git a/ChatQnA/ui/svelte/package.json b/ChatQnA/ui/svelte/package.json index 0f19db6e56..41dbb477b8 100644 --- a/ChatQnA/ui/svelte/package.json +++ b/ChatQnA/ui/svelte/package.json @@ -1,42 +1,35 @@ { - "name": "chat-qna", + "name": "doc-summary", "version": "0.0.1", - "private": true, "scripts": { "dev": "vite dev", "build": "vite build", "preview": "vite preview", + "package": "svelte-kit sync && svelte-package && publint", + "prepublishOnly": "npm run package", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", - "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", - "lint": "prettier --check . && eslint .", - "format": "prettier --write ." + "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch" }, "peerDependencies": { "svelte": "^4.0.0" }, "devDependencies": { - "@fortawesome/free-solid-svg-icons": "6.2.0", - "@playwright/test": "^1.45.2", + "@playwright/test": "^1.44.1", "@sveltejs/adapter-auto": "^3.0.0", "@sveltejs/kit": "^2.0.0", + "@sveltejs/package": "^2.0.0", "@sveltejs/vite-plugin-svelte": "^3.0.0", - "@tailwindcss/typography": "0.5.7", - "@types/debug": "4.1.7", - "@types/node": "^20.12.13", - "@typescript-eslint/eslint-plugin": "^5.27.0", - "@typescript-eslint/parser": "^5.27.0", + "@types/prismjs": "^1.26.3", "autoprefixer": "^10.4.16", - "date-picker-svelte": "^2.6.0", - "debug": "4.3.4", - "postcss": "^8.4.31", - "postcss-load-config": "^4.0.1", - "postcss-preset-env": "^8.3.2", - "prettier": "^2.8.8", - "prettier-plugin-svelte": "^2.7.0", - "prettier-plugin-tailwindcss": "^0.3.0", + "flowbite": "^2.3.0", + "flowbite-svelte": "^0.38.5", + "flowbite-svelte-icons": "^1.4.0", + "postcss": "^8.4.32", + "postcss-load-config": "^5.0.2", + "publint": "^0.1.9", "svelte": "^4.2.7", "svelte-check": "^3.6.0", - "svelte-fa": "3.0.3", + "svelte-highlight": "^7.6.0", "tailwindcss": "^3.3.6", "tslib": "^2.4.1", "typescript": "^5.0.0", @@ -44,17 +37,8 @@ }, "type": "module", "dependencies": { - "date-fns": "^2.30.0", - "driver.js": "^1.3.0", - "flowbite": "^2.5.2", - "flowbite-svelte": "^0.38.5", - "flowbite-svelte-icons": "^1.4.0", - "fuse.js": "^6.6.2", - "lodash": "^4.17.21", - "playwright": "^1.44.0", - "ramda": "^0.29.0", + "prismjs": "^1.29.0", "sse.js": "^0.6.1", - "svelte-notifications": "^0.9.98", - "svrollbar": "^0.12.0" + "svelte-notifications": "^0.9.98" } } diff --git a/ChatQnA/ui/svelte/playwright.config.ts b/ChatQnA/ui/svelte/playwright.config.ts index 937f88bf7b..032caa100f 100644 --- a/ChatQnA/ui/svelte/playwright.config.ts +++ b/ChatQnA/ui/svelte/playwright.config.ts @@ -13,75 +13,42 @@ import { defineConfig, devices } from "@playwright/test"; * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ - testDir: "./tests", - /* Maximum time one test can run for. */ - timeout: 30 * 1000, - expect: { - /** - * Maximum time expect() should wait for the condition to be met. - * For example in `await expect(locator).toHaveText();` - */ - timeout: 5000, - }, - /* Run tests in files in parallel */ - fullyParallel: true, - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, - /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: [["html", { open: "never" }]], - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ - actionTimeout: 0, - /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: "http://localhost:80", - - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: "on-first-retry", - }, - - /* Configure projects for major browsers */ - projects: [ - // { - // name: "chromium", - // use: { ...devices["Desktop Chrome"] }, - // }, - - /* Test against mobile viewports. */ - // { - // name: 'Mobile Chrome', - // use: { ...devices['Pixel 5'] }, - // }, - // { - // name: 'Mobile Safari', - // use: { ...devices['iPhone 12'] }, - // }, - - /* Test against branded browsers. */ - // { - // name: 'Microsoft Edge', - // use: { channel: 'msedge' }, - // }, - { - name: "webkit", - use: { ...devices["Desktop Safari"] }, - }, - // { - // name: 'Google Chrome', - // use: { channel: 'chrome' }, - // }, - ], - - /* Folder for test artifacts such as screenshots, videos, traces, etc. */ - // outputDir: 'test-results/', - - /* Run your local dev server before starting the tests */ - // webServer: { - // command: 'npm run start', - // port: 3000, - // }, + testDir: "./tests", + /* Maximum time one test can run for. */ + timeout: 30 * 1000, + expect: { + /** + * Maximum time expect() should wait for the condition to be met. + * For example in `await expect(locator).toHaveText();` + */ + timeout: 5000, + }, + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: [["html", { open: "never" }]], + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ + actionTimeout: 0, + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: "http://localhost:5173", + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: "on-first-retry", + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: "webkit", + use: { ...devices["Desktop Safari"] }, + }, + ], }); diff --git a/ChatQnA/ui/svelte/postcss.config.cjs b/ChatQnA/ui/svelte/postcss.config.cjs index b384b43ebe..5f822bcb1e 100644 --- a/ChatQnA/ui/svelte/postcss.config.cjs +++ b/ChatQnA/ui/svelte/postcss.config.cjs @@ -16,12 +16,12 @@ const tailwindcss = require("tailwindcss"); const autoprefixer = require("autoprefixer"); const config = { - plugins: [ - //Some plugins, like tailwindcss/nesting, need to run before Tailwind, - tailwindcss(), - //But others, like autoprefixer, need to run after, - autoprefixer, - ], + plugins: [ + //Some plugins, like tailwindcss/nesting, need to run before Tailwind, + tailwindcss(), + //But others, like autoprefixer, need to run after, + autoprefixer, + ], }; module.exports = config; diff --git a/ChatQnA/ui/svelte/src/app.d.ts b/ChatQnA/ui/svelte/src/app.d.ts index fa6a0abf77..1b9de033b6 100644 --- a/ChatQnA/ui/svelte/src/app.d.ts +++ b/ChatQnA/ui/svelte/src/app.d.ts @@ -12,8 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -// See: https://kit.svelte.dev/docs/types#app -// import { Result} from "neverthrow"; -interface Window { - deviceType: string; +// See https://kit.svelte.dev/docs/types#app +// for information about these interfaces +declare global { + namespace App { + // interface Error {} + // interface Locals {} + // interface PageData {} + // interface PageState {} + // interface Platform {} + } } + +export {}; diff --git a/ChatQnA/ui/svelte/src/app.html b/ChatQnA/ui/svelte/src/app.html index db69926ea8..cdcef542df 100644 --- a/ChatQnA/ui/svelte/src/app.html +++ b/ChatQnA/ui/svelte/src/app.html @@ -16,13 +16,13 @@ - - - - - %sveltekit.head% - - -
%sveltekit.body%
- + + + + + %sveltekit.head% + + +
%sveltekit.body%
+ diff --git a/ChatQnA/ui/svelte/src/app.pcss b/ChatQnA/ui/svelte/src/app.pcss new file mode 100644 index 0000000000..1a7b7cf38b --- /dev/null +++ b/ChatQnA/ui/svelte/src/app.pcss @@ -0,0 +1,4 @@ +/* Write your global styles here, in PostCSS syntax */ +@tailwind base; +@tailwind components; +@tailwind utilities; diff --git a/ChatQnA/ui/svelte/src/lib/assets/loadingAnimation.svelte b/ChatQnA/ui/svelte/src/lib/assets/loadingAnimation.svelte new file mode 100644 index 0000000000..713eccc54e --- /dev/null +++ b/ChatQnA/ui/svelte/src/lib/assets/loadingAnimation.svelte @@ -0,0 +1,48 @@ + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/ChatQnA/ui/svelte/src/lib/assets/translateIcon.svelte b/ChatQnA/ui/svelte/src/lib/assets/translateIcon.svelte new file mode 100644 index 0000000000..158dcce980 --- /dev/null +++ b/ChatQnA/ui/svelte/src/lib/assets/translateIcon.svelte @@ -0,0 +1,19 @@ + + + diff --git a/ChatQnA/ui/svelte/src/lib/header.svelte b/ChatQnA/ui/svelte/src/lib/header.svelte new file mode 100644 index 0000000000..7dbf0f1335 --- /dev/null +++ b/ChatQnA/ui/svelte/src/lib/header.svelte @@ -0,0 +1,32 @@ + + + +
+ +
diff --git a/ChatQnA/ui/svelte/src/lib/shared/Network.ts b/ChatQnA/ui/svelte/src/lib/shared/Network.ts new file mode 100644 index 0000000000..45a8fbfd76 --- /dev/null +++ b/ChatQnA/ui/svelte/src/lib/shared/Network.ts @@ -0,0 +1,32 @@ +// Copyright (c) 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +import { env } from "$env/dynamic/public"; +import { SSE } from "sse.js"; + +const BASE_URL = env.BASE_URL; + +export async function fetchTextStream(query: string, langFrom, langTo) { + const payload = { + language_from: langFrom, + language_to: langTo, + source_code: query, + }; + + let url = `${BASE_URL}`; + + return new SSE(url, { + headers: { "Content-Type": "application/json" }, + payload: JSON.stringify(payload), + }); +} diff --git a/ChatQnA/ui/svelte/src/lib/shared/constant.ts b/ChatQnA/ui/svelte/src/lib/shared/constant.ts new file mode 100644 index 0000000000..ca880224d9 --- /dev/null +++ b/ChatQnA/ui/svelte/src/lib/shared/constant.ts @@ -0,0 +1,38 @@ +// Copyright (c) 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { Language } from "../../routes/types.js"; + +export const languagesList: Language[] = [ + { name: "C" }, + { name: "C++" }, + // { name: "C#" }, + { name: "Java" }, + { name: "Python" }, + { name: "JavaScript" }, + // { name: "Swift" }, + // { name: "Ruby" }, + { name: "Go" }, + { name: "Rust" }, + // { name: "PHP" }, + // { name: "TypeScript" }, + // { name: "Kotlin" }, + // { name: "Objective-C" }, + // { name: "Perl" }, + // { name: "MATLAB" }, + // { name: "R" }, + // { name: "Lua" }, + // { name: "Bash" }, + // { name: "SQL" }, +]; diff --git a/ChatQnA/ui/svelte/src/routes/+layout.svelte b/ChatQnA/ui/svelte/src/routes/+layout.svelte index 8141177d4a..9a09eaadfa 100644 --- a/ChatQnA/ui/svelte/src/routes/+layout.svelte +++ b/ChatQnA/ui/svelte/src/routes/+layout.svelte @@ -15,34 +15,11 @@ --> - -
- -
- - + diff --git a/ChatQnA/ui/svelte/src/routes/+page.svelte b/ChatQnA/ui/svelte/src/routes/+page.svelte index b6f6d9c334..fd6be39310 100644 --- a/ChatQnA/ui/svelte/src/routes/+page.svelte +++ b/ChatQnA/ui/svelte/src/routes/+page.svelte @@ -15,297 +15,245 @@ --> - scrollToBottom(scrollToDiv); - storeMessages(); - }; + + {@html atomOneDark} + + +
+
+
+
+
+ + Select Language +
+
+ + + +
+ + +
+ {#if inputClick} +