From da8c92e7c5c855b660a44569012dee71b513770c Mon Sep 17 00:00:00 2001 From: letonghan Date: Mon, 16 Dec 2024 16:53:24 +0800 Subject: [PATCH 1/4] update retriever redis multimodal docker file path Signed-off-by: letonghan --- MultimodalQnA/docker_compose/intel/cpu/xeon/README.md | 2 +- MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md | 2 +- docker_images_list.md | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md b/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md index dcd145bbd7..04dddd0f64 100644 --- a/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md +++ b/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md @@ -124,7 +124,7 @@ docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_pr ### 2. Build retriever-multimodal-redis Image ```bash -docker build --no-cache -t opea/retriever-multimodal-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/multimodal/redis/langchain/Dockerfile . +docker build --no-cache -t opea/retriever-multimodal-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile . ``` ### 3. Build LVM Images diff --git a/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md b/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md index 67e95cb51b..3bed25ffe6 100644 --- a/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md +++ b/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md @@ -75,7 +75,7 @@ docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_pr ### 2. Build retriever-multimodal-redis Image ```bash -docker build --no-cache -t opea/retriever-multimodal-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/multimodal/redis/langchain/Dockerfile . +docker build --no-cache -t opea/retriever-multimodal-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile . ``` ### 3. Build LVM Images diff --git a/docker_images_list.md b/docker_images_list.md index 9efea72ebe..eefddde1c1 100644 --- a/docker_images_list.md +++ b/docker_images_list.md @@ -91,7 +91,6 @@ Take ChatQnA for example. ChatQnA is a chatbot application service based on the | [opea/reranking-langchain-mosec-endpoint](https://hub.docker.com/r/opea/reranking-langchain-mosec-endpoint) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/mosec/langchain/dependency/Dockerfile) | The docker image exposed the OPEA mosec reranking endpoint microservice base on Langchain framework for GenAI application use | | [opea/reranking-tei](https://hub.docker.com/r/opea/reranking-tei) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/tei/Dockerfile) | The docker image exposed the OPEA reranking microservice based on tei docker image for GenAI application use | | [opea/retriever-milvus](https://hub.docker.com/r/opea/retriever-milvus) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/milvus/langchain/Dockerfile) | The docker image exposed the OPEA retrieval microservice based on milvus vectordb for GenAI application use | -| [opea/retriever-multimodal-redis](https://hub.docker.com/r/opea/retriever-multimodal-redis) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/multimodal/redis/langchain/Dockerfile) | The docker image exposed the OPEA retrieval microservice based on multimodal redis vectordb for GenAI application use | | [opea/retriever-pathway](https://hub.docker.com/r/opea/retriever-pathway) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/pathway/langchain/Dockerfile) | The docker image exposed the OPEA retrieval microservice with pathway for GenAI application use | | [opea/retriever-pgvector](https://hub.docker.com/r/opea/retriever-pgvector) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/pgvector/langchain/Dockerfile) | The docker image exposed the OPEA retrieval microservice based on pgvector vectordb for GenAI application use | | [opea/retriever-pinecone](https://hub.docker.com/r/opea/retriever-pinecone) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/pinecone/langchain/Dockerfile) | The docker image exposed the OPEA retrieval microservice based on pinecone vectordb for GenAI application use | From f17e3e53dca43159e43133d53386693131701fbf Mon Sep 17 00:00:00 2001 From: letonghan Date: Tue, 17 Dec 2024 10:15:26 +0800 Subject: [PATCH 2/4] update image name Signed-off-by: letonghan --- MultimodalQnA/docker_compose/intel/cpu/xeon/README.md | 2 +- .../docker_compose/intel/cpu/xeon/compose.yaml | 8 ++++---- MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md | 2 +- .../docker_compose/intel/hpu/gaudi/compose.yaml | 8 ++++---- MultimodalQnA/docker_image_build/build.yaml | 6 +++--- MultimodalQnA/tests/test_compose_on_gaudi.sh | 10 +++++----- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md b/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md index 04dddd0f64..e68538e06d 100644 --- a/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md +++ b/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md @@ -124,7 +124,7 @@ docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_pr ### 2. Build retriever-multimodal-redis Image ```bash -docker build --no-cache -t opea/retriever-multimodal-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile . +docker build --no-cache -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile . ``` ### 3. Build LVM Images diff --git a/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml b/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml index 4f9900db72..7b4f383464 100644 --- a/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -74,9 +74,9 @@ services: MMEI_EMBEDDING_ENDPOINT: ${MMEI_EMBEDDING_ENDPOINT} MM_EMBEDDING_PORT_MICROSERVICE: ${MM_EMBEDDING_PORT_MICROSERVICE} restart: unless-stopped - retriever-multimodal-redis: - image: ${REGISTRY:-opea}/retriever-multimodal-redis:${TAG:-latest} - container_name: retriever-multimodal-redis + retriever-redis: + image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} + container_name: retriever-redis depends_on: - redis-vector-db ports: @@ -121,7 +121,7 @@ services: - redis-vector-db - dataprep-multimodal-redis - embedding-multimodal - - retriever-multimodal-redis + - retriever-redis - lvm-llava-svc - asr ports: diff --git a/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md b/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md index 3bed25ffe6..b8c5e3bcd2 100644 --- a/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md +++ b/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md @@ -75,7 +75,7 @@ docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_pr ### 2. Build retriever-multimodal-redis Image ```bash -docker build --no-cache -t opea/retriever-multimodal-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile . +docker build --no-cache -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile . ``` ### 3. Build LVM Images diff --git a/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 87a6821c77..6f8b649775 100644 --- a/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -74,9 +74,9 @@ services: MMEI_EMBEDDING_ENDPOINT: ${MMEI_EMBEDDING_ENDPOINT} MM_EMBEDDING_PORT_MICROSERVICE: ${MM_EMBEDDING_PORT_MICROSERVICE} restart: unless-stopped - retriever-multimodal-redis: - image: ${REGISTRY:-opea}/retriever-multimodal-redis:${TAG:-latest} - container_name: retriever-multimodal-redis + retriever-redis: + image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} + container_name: retriever-redis depends_on: - redis-vector-db ports: @@ -138,7 +138,7 @@ services: - redis-vector-db - dataprep-multimodal-redis - embedding-multimodal - - retriever-multimodal-redis + - retriever-redis - lvm-tgi - asr ports: diff --git a/MultimodalQnA/docker_image_build/build.yaml b/MultimodalQnA/docker_image_build/build.yaml index 0efa6a37e3..efb2f656d8 100644 --- a/MultimodalQnA/docker_image_build/build.yaml +++ b/MultimodalQnA/docker_image_build/build.yaml @@ -29,12 +29,12 @@ services: dockerfile: comps/embeddings/multimodal/multimodal_langchain/Dockerfile extends: multimodalqna image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest} - retriever-multimodal-redis: + retriever-redis: build: context: GenAIComps - dockerfile: comps/retrievers/multimodal/redis/langchain/Dockerfile + dockerfile: comps/retrievers/redis/langchain/Dockerfile extends: multimodalqna - image: ${REGISTRY:-opea}/retriever-multimodal-redis:${TAG:-latest} + image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} lvm-llava: build: context: GenAIComps diff --git a/MultimodalQnA/tests/test_compose_on_gaudi.sh b/MultimodalQnA/tests/test_compose_on_gaudi.sh index 0c522dd08f..9039a46284 100644 --- a/MultimodalQnA/tests/test_compose_on_gaudi.sh +++ b/MultimodalQnA/tests/test_compose_on_gaudi.sh @@ -22,7 +22,7 @@ function build_docker_images() { cd $WORKPATH/docker_image_build git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding-multimodal retriever-multimodal-redis lvm-tgi dataprep-multimodal-redis whisper asr" + service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding-multimodal retriever-redis lvm-tgi dataprep-multimodal-redis whisper asr" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 @@ -192,13 +192,13 @@ function validate_microservices() { sleep 1m # multimodal retrieval microservice - echo "Validating retriever-multimodal-redis" + echo "Validating retriever-redis" your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") validate_service \ - "http://${host_ip}:7000/v1/multimodal_retrieval" \ + "http://${host_ip}:7000/v1/retrieval" \ "retrieved_docs" \ - "retriever-multimodal-redis" \ - "retriever-multimodal-redis" \ + "retriever-redis" \ + "retriever-redis" \ "{\"text\":\"test\",\"embedding\":${your_embedding}}" sleep 3m From 9a61e5ae36853577a6fb639893e5739e310622ba Mon Sep 17 00:00:00 2001 From: letonghan Date: Tue, 17 Dec 2024 10:21:24 +0800 Subject: [PATCH 3/4] fix in test script Signed-off-by: letonghan --- MultimodalQnA/tests/test_compose_on_xeon.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/MultimodalQnA/tests/test_compose_on_xeon.sh b/MultimodalQnA/tests/test_compose_on_xeon.sh index 0475272206..4c3b5572d7 100644 --- a/MultimodalQnA/tests/test_compose_on_xeon.sh +++ b/MultimodalQnA/tests/test_compose_on_xeon.sh @@ -22,7 +22,7 @@ function build_docker_images() { cd $WORKPATH/docker_image_build git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding-multimodal retriever-multimodal-redis lvm-llava lvm-llava-svc dataprep-multimodal-redis whisper asr" + service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding-multimodal retriever-redis lvm-llava lvm-llava-svc dataprep-multimodal-redis whisper asr" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log docker images && sleep 1m @@ -190,13 +190,13 @@ function validate_microservices() { sleep 1m # multimodal retrieval microservice - echo "Validating retriever-multimodal-redis" + echo "Validating retriever-redis" your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") validate_service \ - "http://${host_ip}:7000/v1/multimodal_retrieval" \ + "http://${host_ip}:7000/v1/retrieval" \ "retrieved_docs" \ - "retriever-multimodal-redis" \ - "retriever-multimodal-redis" \ + "retriever-redis" \ + "retriever-redis" \ "{\"text\":\"test\",\"embedding\":${your_embedding}}" sleep 3m From 7c08862e5664025c4397e7278efe2001caaa540b Mon Sep 17 00:00:00 2001 From: letonghan Date: Tue, 17 Dec 2024 16:58:35 +0800 Subject: [PATCH 4/4] fix multimodal retriever issues Signed-off-by: letonghan --- MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml | 1 + MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 1 + MultimodalQnA/multimodalqna.py | 2 +- MultimodalQnA/tests/test_compose_on_gaudi.sh | 1 + MultimodalQnA/tests/test_compose_on_xeon.sh | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml b/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml index 7b4f383464..884fa9276a 100644 --- a/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -88,6 +88,7 @@ services: https_proxy: ${https_proxy} REDIS_URL: ${REDIS_URL} INDEX_NAME: ${INDEX_NAME} + BRIDGE_TOWER_EMBEDDING: ${BRIDGE_TOWER_EMBEDDING} restart: unless-stopped lvm-llava: image: ${REGISTRY:-opea}/lvm-llava:${TAG:-latest} diff --git a/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 6f8b649775..1517d6f110 100644 --- a/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -88,6 +88,7 @@ services: https_proxy: ${https_proxy} REDIS_URL: ${REDIS_URL} INDEX_NAME: ${INDEX_NAME} + BRIDGE_TOWER_EMBEDDING: ${BRIDGE_TOWER_EMBEDDING} restart: unless-stopped tgi-gaudi: image: ghcr.io/huggingface/tgi-gaudi:2.0.6 diff --git a/MultimodalQnA/multimodalqna.py b/MultimodalQnA/multimodalqna.py index fb70128613..c64012224f 100644 --- a/MultimodalQnA/multimodalqna.py +++ b/MultimodalQnA/multimodalqna.py @@ -54,7 +54,7 @@ def add_remote_service(self): name="retriever", host=MM_RETRIEVER_SERVICE_HOST_IP, port=MM_RETRIEVER_SERVICE_PORT, - endpoint="/v1/multimodal_retrieval", + endpoint="/v1/retrieval", use_remote_service=True, service_type=ServiceType.RETRIEVER, ) diff --git a/MultimodalQnA/tests/test_compose_on_gaudi.sh b/MultimodalQnA/tests/test_compose_on_gaudi.sh index 9039a46284..4b8591423b 100644 --- a/MultimodalQnA/tests/test_compose_on_gaudi.sh +++ b/MultimodalQnA/tests/test_compose_on_gaudi.sh @@ -41,6 +41,7 @@ function setup_env() { export REDIS_URL="redis://${host_ip}:6379" export REDIS_HOST=${host_ip} export INDEX_NAME="mm-rag-redis" + export BRIDGE_TOWER_EMBEDDING=true export LLAVA_SERVER_PORT=8399 export LVM_ENDPOINT="http://${host_ip}:8399" export EMBEDDING_MODEL_ID="BridgeTower/bridgetower-large-itm-mlm-itc" diff --git a/MultimodalQnA/tests/test_compose_on_xeon.sh b/MultimodalQnA/tests/test_compose_on_xeon.sh index 4c3b5572d7..101f76e98b 100644 --- a/MultimodalQnA/tests/test_compose_on_xeon.sh +++ b/MultimodalQnA/tests/test_compose_on_xeon.sh @@ -39,6 +39,7 @@ function setup_env() { export REDIS_URL="redis://${host_ip}:6379" export REDIS_HOST=${host_ip} export INDEX_NAME="mm-rag-redis" + export BRIDGE_TOWER_EMBEDDING=true export LLAVA_SERVER_PORT=8399 export LVM_ENDPOINT="http://${host_ip}:8399" export LVM_MODEL_ID="llava-hf/llava-1.5-7b-hf"