[Bug: 1379] Added Multimodal support for Milvus for retriever component (opea-project#1381)

Added Multimodal support for Milvus for retriever component

Fixes opea-project#1379

Co-authored-by: Jaini, Pallavi <pallavi.jaini@intel.com>
Co-authored-by: Kavulya, Soila <soila.kavulya@intel.com>
Signed-off-by:  Ghosh, Soumyadip <soumyadip.ghosh@intel.com>
Signed-off-by:  Piroozan, Nariman <nariman.piroozan@intel.com>

Signed-off-by: Shifani Rajabose <srajabose@habana.ai>
Signed-off-by: pallavi.jaini <pallavi.jaini@intel.com>

Author: srajabos

comps/retrievers/deployment/docker_compose/compose.yaml

@@ -57,6 +57,23 @@ services:
       minio:
         condition: service_healthy
 
+  retriever-milvus-multimodal:
+    extends: retriever
+    container_name: retriever-milvus-multimodal
+    environment:
+      RETRIEVER_COMPONENT_NAME: ${RETRIEVER_COMPONENT_NAME:-OPEA_RETRIEVER_MILVUS}
+      RETRIEVER_TYPE: ${RETRIEVER_TYPE:-milvus}
+      MILVUS_HOST: ${host_ip}
+      BRIDGE_TOWER_EMBEDDING: ${BRIDGE_TOWER_EMBEDDING}
+      COLLECTION_NAME: ${COLLECTION_NAME:-LangChainCollection}
+    depends_on:
+      standalone:
+        condition: service_healthy
+      etcd:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+
   retriever-neo4j:
     extends: retriever
     container_name: retriever-neo4j

comps/retrievers/src/integrations/milvus.py

@@ -12,6 +12,7 @@
 from comps import CustomLogger, EmbedDoc, OpeaComponent, OpeaComponentRegistry, ServiceType
 
 from .config import (
+    BRIDGE_TOWER_EMBEDDING,
     COLLECTION_NAME,
     HUGGINGFACEHUB_API_TOKEN,
     INDEX_PARAMS,
@@ -61,6 +62,11 @@ def _initialize_embedder(self):
             embeddings = HuggingFaceInferenceAPIEmbeddings(
                 api_key=HUGGINGFACEHUB_API_TOKEN, model_name=model_id, api_url=TEI_EMBEDDING_ENDPOINT
             )
+        elif BRIDGE_TOWER_EMBEDDING:
+            logger.info("use bridge tower embedding")
+            from comps.third_parties.bridgetower.src.bridgetower_embedding import BridgeTowerEmbedding
+
+            embeddings = BridgeTowerEmbedding()
         else:
             # create embeddings using local embedding model
             if logflag:

tests/retrievers/test_retrievers_milvus.sh

@@ -16,6 +16,7 @@ LOG_PATH="$WORKPATH/tests"
 export host_ip=$(hostname -I | awk '{print $1}')
 service_name="retriever-milvus etcd minio standalone"
 retriever_service_name="retriever-milvus"
+service_name_mm="retriever-milvus-multimodal etcd minio standalone"
 
 function build_docker_images() {
     cd $WORKPATH
@@ -46,6 +47,23 @@ function start_service() {
     sleep 1m
 }
 
+function start_multimodal_service() {
+    export MINIO_PORT1=11611
+    export MINIO_PORT2=11612
+    export MILVUS_STANDALONE_PORT=11613
+    export TEI_EMBEDDER_PORT=11614
+    export HF_TOKEN=${HF_TOKEN}
+    export COLLECTION_NAME="LangChainCollection"
+    export LOGFLAG=True
+    export BRIDGE_TOWER_EMBEDDING=true
+    export RETRIEVER_TYPE="milvus"
+
+    cd $WORKPATH/comps/retrievers/deployment/docker_compose
+    docker compose -f compose.yaml up ${service_name_mm} -d > ${LOG_PATH}/start_services_with_compose_multimodal.log
+
+    sleep 2m
+}
+
 function validate_microservice() {
     local test_embedding="$1"
 
@@ -72,12 +90,57 @@ function validate_microservice() {
     fi
 }
 
+function validate_mm_microservice() {
+    local test_embedding="$1"
+    local container_name="$2"
+
+    export PATH="${HOME}/miniforge3/bin:$PATH"
+    source activate
+    URL="http://${host_ip}:$RETRIEVER_PORT/v1/retrieval"
+
+    # Test the retriever with a b64 image that should be passed through
+    HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "{\"text\":\"test\",\"embedding\":${test_embedding},\"base64_image\":\"iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC\"}" -H 'Content-Type: application/json' "$URL")
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ retriever ] HTTP status is 200. Checking content..."
+        local CONTENT=$(curl -s -X POST -d "{\"text\":\"test\",\"embedding\":${test_embedding}}" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/retriever.log)
+
+        if echo "$CONTENT" | grep -q "retrieved_docs"; then
+            echo "[ retriever ] Content has retrieved_docs as expected."
+            empty_search_results=$(echo "$CONTENT" | grep "\"retrieved_docs\":\[\]")
+            if [ -z "$empty_search_results" ]; then
+                # If search results are not empty, check for b64 image string
+                if echo "$CONTENT" | grep -q "b64_img_str"; then
+                    echo "[ retriever ] Content has b64_img_str as expected."
+                else
+                    echo "[ retriever ] Content does not include the b64_img_str: $CONTENT"
+                    docker logs ${container_name} >> ${LOG_PATH}/retriever.log
+                    exit 1
+                fi
+            fi
+        else
+            echo "[ retriever ] Content does not match the expected result: $CONTENT"
+            docker logs ${container_name} >> ${LOG_PATH}/retriever.log
+            exit 1
+        fi
+    else
+        echo "[ retriever ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${container_name} >> ${LOG_PATH}/retriever.log
+        exit 1
+    fi
+}
+
 function stop_docker() {
     cd $WORKPATH/comps/third_parties/milvus/deployment/docker_compose/
     docker compose -f compose.yaml down --remove-orphans
 
+    docker stop milvus-etcd
+    docker rm milvus-etcd
+
     cd $WORKPATH/comps/retrievers/deployment/docker_compose
-    docker compose -f compose.yaml down --remove-orphans
+
+    docker compose -f compose.yaml down  ${service_name} --remove-orphans
+    docker compose -f compose.yaml down ${service_name_mm} --remove-orphans
+
     cid=$(docker ps -aq --filter "name=tei-embedding-serving")
     if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
 }
@@ -92,6 +155,14 @@ function main() {
     validate_microservice "$test_embedding"
 
     stop_docker
+
+    start_multimodal_service
+    test_embedding_multi=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)")
+    validate_microservice "$test_embedding_multi" "$service_name_mm"
+    validate_mm_microservice "$test_embedding_multi" "$service_name_mm"
+
+    stop_docker
+
     echo y | docker system prune
 
 }