Merge branch 'main' into feature/GenAIExample_ChatQnA_AMD_vLLM

Author: chyundunovDatamonsters

ChatQnA/benchmark/performance/kubernetes/intel/gaudi/README.md

@@ -69,10 +69,6 @@ Results will be displayed in the terminal and saved as CSV file named `1_stats.c
   - Persistent Volume Claim (PVC): This is the recommended approach for production setups. For more details on using PVC, refer to [PVC](https://github.com/opea-project/GenAIInfra/blob/main/helm-charts/README.md#using-persistent-volume).
   - Local Host Path: For simpler testing, ensure that each node involved in the deployment follows the steps above to locally prepare the models. After preparing the models, use `--set global.modelUseHostPath=${MODELDIR}` in the deployment command.
 
-- Add OPEA Helm Repository:
-  ```bash
-  python deploy.py --add-repo
-  ```
 - Label Nodes
   ```base
   python deploy.py --add-label --num-nodes 2
@@ -192,13 +188,9 @@ All the test results will come to the folder `GenAIEval/evals/benchmark/benchmar
 
 ## Teardown
 
-After completing the benchmark, use the following commands to clean up the environment:
+After completing the benchmark, use the following command to clean up the environment:
 
 Remove Node Labels:
-```base
-python deploy.py --delete-label
-```
-Delete the OPEA Helm Repository:
 ```bash
-python deploy.py --delete-repo
+python deploy.py --delete-label
 ```

ChatQnA/benchmark/performance/kubernetes/intel/gaudi/deploy.py

@@ -83,26 +83,6 @@ def clear_labels_from_nodes(label, node_names=None):
             print(f"Label {label_key} not found on node {node_name}, skipping.")
 
 
-def add_helm_repo(repo_name, repo_url):
-    # Add the repo if it does not exist
-    add_command = ["helm", "repo", "add", repo_name, repo_url]
-    try:
-        subprocess.run(add_command, check=True)
-        print(f"Added Helm repo {repo_name} from {repo_url}.")
-    except subprocess.CalledProcessError as e:
-        print(f"Failed to add Helm repo {repo_name}: {e}")
-
-
-def delete_helm_repo(repo_name):
-    """Delete Helm repo if it exists."""
-    command = ["helm", "repo", "remove", repo_name]
-    try:
-        subprocess.run(command, check=True)
-        print(f"Deleted Helm repo {repo_name}.")
-    except subprocess.CalledProcessError:
-        print(f"Failed to delete Helm repo {repo_name}. It may not exist.")
-
-
 def install_helm_release(release_name, chart_name, namespace, values_file, device_type):
     """Deploy a Helm release with a specified name and chart.
 
@@ -132,14 +112,14 @@ def install_helm_release(release_name, chart_name, namespace, values_file, devic
     if device_type == "gaudi":
         print("Device type is gaudi. Pulling Helm chart to get gaudi-values.yaml...")
 
-        # Pull and untar the chart
-        subprocess.run(["helm", "pull", chart_name, "--untar"], check=True)
+        # Combine chart_name with fixed prefix
+        chart_pull_url = f"oci://ghcr.io/opea-project/charts/{chart_name}"
 
-        # Determine the directory name (get the actual chart_name if chart_name is in the format 'repo_name/chart_name', else use chart_name directly)
-        chart_dir_name = chart_name.split("/")[-1] if "/" in chart_name else chart_name
+        # Pull and untar the chart
+        subprocess.run(["helm", "pull", chart_pull_url, "--untar"], check=True)
 
-        # Find the untarred directory (assumes only one directory matches chart_dir_name)
-        untar_dirs = glob.glob(f"{chart_dir_name}*")
+        # Find the untarred directory
+        untar_dirs = glob.glob(f"{chart_name}*")
         if untar_dirs:
             untar_dir = untar_dirs[0]
             hw_values_file = os.path.join(untar_dir, "gaudi-values.yaml")
@@ -210,20 +190,14 @@ def main():
     parser.add_argument(
         "--chart-name",
         type=str,
-        default="opea/chatqna",
-        help="The chart name to deploy, composed of repo name and chart name (default: opea/chatqna).",
+        default="chatqna",
+        help="The chart name to deploy, composed of repo name and chart name (default: chatqna).",
     )
     parser.add_argument("--namespace", default="default", help="Kubernetes namespace (default: default).")
     parser.add_argument("--hf-token", help="Hugging Face API token.")
     parser.add_argument(
         "--model-dir", help="Model directory, mounted as volumes for service access to pre-downloaded models"
     )
-    parser.add_argument("--repo-name", default="opea", help="Helm repo name to add/delete (default: opea).")
-    parser.add_argument(
-        "--repo-url",
-        default="https://opea-project.github.io/GenAIInfra",
-        help="Helm repository URL (default: https://opea-project.github.io/GenAIInfra).",
-    )
     parser.add_argument("--user-values", help="Path to a user-specified values.yaml file.")
     parser.add_argument(
         "--create-values-only", action="store_true", help="Only create the values.yaml file without deploying."
@@ -244,8 +218,6 @@ def main():
         action="store_true",
         help="Modify resources for services and change extraCmdArgs when creating values.yaml.",
     )
-    parser.add_argument("--add-repo", action="store_true", help="Add the Helm repo specified by --repo-url.")
-    parser.add_argument("--delete-repo", action="store_true", help="Delete the Helm repo specified by --repo-name.")
     parser.add_argument(
         "--device-type",
         type=str,
@@ -264,14 +236,6 @@ def main():
         else:
             args.num_nodes = num_node_names
 
-    # Helm repository management
-    if args.add_repo:
-        add_helm_repo(args.repo_name, args.repo_url)
-        return
-    elif args.delete_repo:
-        delete_helm_repo(args.repo_name)
-        return
-
     # Node labeling management
     if args.add_label:
         add_labels_to_nodes(args.num_nodes, args.label, args.node_names)

DocIndexRetriever/tests/test.py

@@ -6,7 +6,7 @@
 import requests
 
 
-def search_knowledge_base(query: str, url: str, request_type="chat_completion") -> str:
+def search_knowledge_base(query: str, url: str, request_type: str) -> str:
     """Search the knowledge base for a specific query."""
     print(url)
     proxies = {"http": ""}
@@ -18,12 +18,13 @@ def search_knowledge_base(query: str, url: str, request_type="chat_completion")
             "top_n": 2,
         }
     else:
-        print("Sending text request")
+        print("Sending textdoc request")
         payload = {
             "text": query,
         }
     response = requests.post(url, json=payload, proxies=proxies)
     print(response)
+    print(response.json().keys())
     if "documents" in response.json():
         docs = response.json()["documents"]
         context = ""
@@ -32,7 +33,6 @@ def search_knowledge_base(query: str, url: str, request_type="chat_completion")
                 context = str(i) + ": " + doc
             else:
                 context += "\n" + str(i) + ": " + doc
-        # print(context)
         return context
     elif "text" in response.json():
         return response.json()["text"]
@@ -44,7 +44,6 @@ def search_knowledge_base(query: str, url: str, request_type="chat_completion")
                 context = doc["text"]
             else:
                 context += "\n" + doc["text"]
-        # print(context)
         return context
     else:
         return "Error parsing response from the knowledge base."

DocIndexRetriever/tests/test_compose_on_gaudi.sh

@@ -15,6 +15,7 @@ LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
+    echo "Building Docker Images...."
     cd $WORKPATH/docker_image_build
     if [ ! -d "GenAIComps" ] ; then
         git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
@@ -26,9 +27,11 @@ function build_docker_images() {
     docker pull redis/redis-stack:7.2.0-v9
     docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
     docker images && sleep 1s
+    echo "Docker images built!"
 }
 
 function start_services() {
+    echo "Starting Docker Services...."
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export RERANK_MODEL_ID="BAAI/bge-reranker-base"
@@ -47,6 +50,7 @@ function start_services() {
     # Start Docker Containers
     docker compose up -d
     sleep 20
+    echo "Docker services started!"
 }
 
 function validate() {

DocIndexRetriever/tests/test_compose_on_xeon.sh

@@ -15,8 +15,10 @@ LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
+    echo "Building Docker Images...."
     cd $WORKPATH/docker_image_build
     if [ ! -d "GenAIComps" ] ; then
+        echo "Cloning GenAIComps repository"
         git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
     fi
     service_list="dataprep-redis embedding-tei retriever-redis reranking-tei doc-index-retriever"
@@ -25,9 +27,12 @@ function build_docker_images() {
     docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     docker pull redis/redis-stack:7.2.0-v9
     docker images && sleep 1s
+
+    echo "Docker images built!"
 }
 
 function start_services() {
+    echo "Starting Docker Services...."
     cd $WORKPATH/docker_compose/intel/cpu/xeon
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export RERANK_MODEL_ID="BAAI/bge-reranker-base"
@@ -45,7 +50,8 @@ function start_services() {
 
     # Start Docker Containers
     docker compose up -d
-    sleep 20
+    sleep 5m
+    echo "Docker services started!"
 }
 
 function validate() {
@@ -66,7 +72,7 @@ function validate_megaservice() {
     echo "===========Ingest data=================="
     local CONTENT=$(http_proxy="" curl -X POST "http://${ip_address}:6007/v1/dataprep" \
      -H "Content-Type: multipart/form-data" \
-     -F 'link_list=["https://opea.dev"]')
+     -F 'link_list=["https://opea.dev/"]')
     local EXIT_CODE=$(validate "$CONTENT" "Data preparation succeeded" "dataprep-redis-service-xeon")
     echo "$EXIT_CODE"
     local EXIT_CODE="${EXIT_CODE:0-1}"
@@ -77,19 +83,26 @@ function validate_megaservice() {
     fi
 
     # Curl the Mega Service
-    echo "================Testing retriever service: Default params================"
-
-    local CONTENT=$(curl http://${ip_address}:8889/v1/retrievaltool -X POST -H "Content-Type: application/json" -d '{
-     "messages": "Explain the OPEA project?"
+    echo "================Testing retriever service: Text Request ================"
+    cd $WORKPATH/tests
+    local CONTENT=$(http_proxy="" curl http://${ip_address}:8889/v1/retrievaltool -X POST -H "Content-Type: application/json" -d '{
+     "text": "Explain the OPEA project?"
     }')
+    # local CONTENT=$(python test.py --host_ip ${ip_address} --request_type text)
     local EXIT_CODE=$(validate "$CONTENT" "OPEA" "doc-index-retriever-service-xeon")
     echo "$EXIT_CODE"
     local EXIT_CODE="${EXIT_CODE:0-1}"
     echo "return value is $EXIT_CODE"
     if [ "$EXIT_CODE" == "1" ]; then
-        docker logs tei-embedding-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============Embedding container log=================="
+        docker logs embedding-tei-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============Retriever container log=================="
         docker logs retriever-redis-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
-        docker logs reranking-tei-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============TEI Reranking log=================="
+        docker logs tei-reranking-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============Reranking container log=================="
+        docker logs reranking-tei-xeon-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============Doc-index-retriever container log=================="
         docker logs doc-index-retriever-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
         exit 1
     fi
@@ -102,9 +115,15 @@ function validate_megaservice() {
     local EXIT_CODE="${EXIT_CODE:0-1}"
     echo "return value is $EXIT_CODE"
     if [ "$EXIT_CODE" == "1" ]; then
-        docker logs tei-embedding-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============Embedding container log=================="
+        docker logs embedding-tei-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============Retriever container log=================="
         docker logs retriever-redis-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
-        docker logs reranking-tei-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============TEI Reranking log=================="
+        docker logs tei-reranking-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============Reranking container log=================="
+        docker logs reranking-tei-xeon-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
+        echo "=============Doc-index-retriever container log=================="
         docker logs doc-index-retriever-server | tee -a ${LOG_PATH}/doc-index-retriever-service-xeon.log
         exit 1
     fi

EdgeCraftRAG/Dockerfile.server

@@ -23,6 +23,11 @@ RUN useradd -m -s /bin/bash user && \
     mkdir -p /home/user && \
     chown -R user /home/user/ 
 
+RUN mkdir /templates && \
+    chown -R user /templates
+COPY ./edgecraftrag/prompt_template/default_prompt.txt /templates/
+RUN chown -R user /templates/default_prompt.txt
+
 COPY ./edgecraftrag /home/user/edgecraftrag
 
 RUN mkdir -p /home/user/gradio_cache 

EdgeCraftRAG/README.md

@@ -32,14 +32,14 @@ Please follow this link [vLLM with OpenVINO](https://github.com/opea-project/Gen
 
 ### Start Edge Craft RAG Services with Docker Compose
 
-If you want to enable vLLM with OpenVINO service, please finish the steps in [Launch vLLM with OpenVINO service](#optional-launch-vllm-with-openvino-service) first.
-
 ```bash
 cd GenAIExamples/EdgeCraftRAG/docker_compose/intel/gpu/arc
 
 export MODEL_PATH="your model path for all your models"
 export DOC_PATH="your doc path for uploading a dir of files"
 export GRADIO_PATH="your gradio cache path for transferring files"
+# If you have a specific prompt template, please uncomment the following line
+# export PROMPT_PATH="your prompt path for prompt templates"
 
 # Make sure all 3 folders have 1000:1000 permission, otherwise
 # chown 1000:1000 ${MODEL_PATH} ${DOC_PATH} ${GRADIO_PATH}
@@ -70,49 +70,25 @@ optimum-cli export openvino -m BAAI/bge-small-en-v1.5 ${MODEL_PATH}/BAAI/bge-sma
 optimum-cli export openvino -m BAAI/bge-reranker-large ${MODEL_PATH}/BAAI/bge-reranker-large --task sentence-similarity
 optimum-cli export openvino -m Qwen/Qwen2-7B-Instruct ${MODEL_PATH}/Qwen/Qwen2-7B-Instruct/INT4_compressed_weights --weight-format int4
 
-docker compose up -d
+```
+
+#### Launch services with local inference
 
+```bash
+docker compose -f compose.yaml up -d
 ```
 
-#### (Optional) Launch vLLM with OpenVINO service
+#### Launch services with vLLM + OpenVINO inference service
 
-1. Set up Environment Variables
+Set up Additional Environment Variables and start with compose_vllm.yaml
 
 ```bash
 export LLM_MODEL=#your model id
 export VLLM_SERVICE_PORT=8008
 export vLLM_ENDPOINT="http://${HOST_IP}:${VLLM_SERVICE_PORT}"
 export HUGGINGFACEHUB_API_TOKEN=#your HF token
-```
-
-2. Uncomment below code in 'GenAIExamples/EdgeCraftRAG/docker_compose/intel/gpu/arc/compose.yaml'
 
-```bash
-  # vllm-openvino-server:
-  #   container_name: vllm-openvino-server
-  #   image: opea/vllm-arc:latest
-  #   ports:
-  #     - ${VLLM_SERVICE_PORT:-8008}:80
-  #   environment:
-  #     HTTPS_PROXY: ${https_proxy}
-  #     HTTP_PROXY: ${https_proxy}
-  #     VLLM_OPENVINO_DEVICE: GPU
-  #     HF_ENDPOINT: ${HF_ENDPOINT}
-  #     HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
-  #   volumes:
-  #     - /dev/dri/by-path:/dev/dri/by-path
-  #     - $HOME/.cache/huggingface:/root/.cache/huggingface
-  #   devices:
-  #     - /dev/dri
-  #   entrypoint: /bin/bash -c "\
-  #     cd / && \
-  #     export VLLM_CPU_KVCACHE_SPACE=50 && \
-  #     export VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON && \
-  #     python3 -m vllm.entrypoints.openai.api_server \
-  #       --model '${LLM_MODEL}' \
-  #       --max_model_len=1024 \
-  #       --host 0.0.0.0 \
-  #       --port 80"
+docker compose -f compose_vllm.yaml up -d
 ```
 
 ### ChatQnA with LLM Example (Command Line)

EdgeCraftRAG/docker_compose/intel/gpu/arc/compose.yaml

@@ -16,6 +16,7 @@ services:
       - ${DOC_PATH:-${PWD}}:/home/user/docs
       - ${GRADIO_PATH:-${PWD}}:/home/user/gradio_cache
       - ${HF_CACHE:-${HOME}/.cache}:/home/user/.cache
+      - ${PROMPT_PATH:-${PWD}}:/templates/custom
     ports:
       - ${PIPELINE_SERVICE_PORT:-16010}:${PIPELINE_SERVICE_PORT:-16010}
     devices:

EdgeCraftRAG/docker_compose/intel/gpu/arc/compose_vllm.yaml

@@ -16,6 +16,7 @@ services:
       - ${DOC_PATH:-${PWD}}:/home/user/docs
       - ${GRADIO_PATH:-${PWD}}:/home/user/gradio_cache
       - ${HF_CACHE:-${HOME}/.cache}:/home/user/.cache
+      - ${PROMPT_PATH:-${PWD}}:/templates/custom
     ports:
       - ${PIPELINE_SERVICE_PORT:-16010}:${PIPELINE_SERVICE_PORT:-16010}
     devices: