VDMS langchain package update (opea-project#1317)

* Update VDMS related components

* Pinned protobuf==4.24.2 for vdms to avoid issues

* Move opentelemetry installation to requirements file like others, unset proxy in opensearch tests. Opensearch tests pass locally with setting vm.max_map_count

* pin protobuf and downgrade opentelemetry to v1.27.0 in dataprep and retriever dockerfiles

* Move opentelemetry installation to requirements file like others, unset proxy in opensearch tests. Opensearch tests pass locally with setting vm.max_map_count

Signed-off-by: Lacewell, Chaunte W <chaunte.w.lacewell@intel.com>
Co-authored-by: chen, suyue <suyue.chen@intel.com>
Signed-off-by: Chingis Yundunov <c.yundunov@datamonsters.com>

Author: 2 people

.github/workflows/pr-microservice-test.yml

@@ -64,6 +64,10 @@ jobs:
           role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }}
           aws-region: us-east-1
 
+      - name: Set Memory Map Limit
+        if: ${{ contains(matrix.service, "opensearch") }}
+        run: sudo sysctl -w vm.max_map_count=262144
+
       - name: Run microservice test
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}

comps/dataprep/src/Dockerfile

@@ -39,8 +39,7 @@ RUN pip install --no-cache-dir --upgrade pip setuptools && \
         PIP_EXTRA_INDEX_URL=""; \
     fi && \
     pip install --no-cache-dir torch torchvision ${PIP_EXTRA_INDEX_URL} && \
-    pip install --no-cache-dir ${PIP_EXTRA_INDEX_URL} -r /home/user/comps/dataprep/src/requirements.txt && \
-    pip install opentelemetry-api==1.29.0 opentelemetry-exporter-otlp==1.29.0 opentelemetry-sdk==1.29.0
+    pip install --no-cache-dir ${PIP_EXTRA_INDEX_URL} -r /home/user/comps/dataprep/src/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 

comps/dataprep/src/integrations/utils/store_embeddings.py

@@ -6,10 +6,9 @@
 import numpy as np
 import torchvision.transforms as T
 from decord import VideoReader, cpu
-from langchain.pydantic_v1 import BaseModel, root_validator
-from langchain_community.vectorstores import VDMS
-from langchain_community.vectorstores.vdms import VDMS_Client
 from langchain_core.embeddings import Embeddings
+from langchain_vdms.vectorstores import VDMS, VDMS_Client
+from pydantic import BaseModel, model_validator
 
 toPIL = T.ToPILImage()
 
@@ -21,7 +20,7 @@ class vCLIPEmbeddings(BaseModel, Embeddings):
 
     model: Any
 
-    @root_validator(allow_reuse=True)
+    @model_validator(mode="before")
     def validate_environment(cls, values: Dict) -> Dict:
         """Validate that open_clip and torch libraries are installed."""
         try:
@@ -99,6 +98,8 @@ def __init__(
         collection_name,
         embedding_dimensions: int = 512,
         chosen_video_search_type="similarity",
+        engine: str = "FaissFlat",
+        distance_strategy: str = "IP",
     ):
 
         self.host = host
@@ -110,6 +111,8 @@ def __init__(
         self.video_embedder = vCLIPEmbeddings(model=video_retriever_model)
         self.chosen_video_search_type = chosen_video_search_type
         self.embedding_dimensions = embedding_dimensions
+        self.engine = engine
+        self.distance_strategy = distance_strategy
 
         # initialize_db
         self.get_db_client()
@@ -128,7 +131,7 @@ def init_db(self):
                 client=self.client,
                 embedding=self.video_embedder,
                 collection_name=self.video_collection,
-                engine="FaissFlat",
-                distance_strategy="IP",
+                engine=self.engine,
+                distance_strategy=self.distance_strategy,
                 embedding_dimensions=self.embedding_dimensions,
             )

comps/dataprep/src/integrations/vdms.py

@@ -8,9 +8,9 @@
 from fastapi import Body, File, Form, HTTPException, UploadFile
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import HuggingFaceBgeEmbeddings, HuggingFaceInferenceAPIEmbeddings
-from langchain_community.vectorstores.vdms import VDMS, VDMS_Client
 from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_text_splitters import HTMLHeaderTextSplitter
+from langchain_vdms.vectorstores import VDMS, VDMS_Client
 
 from comps import CustomLogger, DocPath, OpeaComponent, OpeaComponentRegistry, ServiceType
 from comps.dataprep.src.utils import (

comps/dataprep/src/integrations/vdms_multimodal.py

@@ -23,6 +23,8 @@
 VECTORDB_SERVICE_HOST_IP = os.getenv("VDMS_HOST", "0.0.0.0")
 VECTORDB_SERVICE_PORT = os.getenv("VDMS_PORT", 55555)
 collection_name = os.getenv("INDEX_NAME", "rag-vdms")
+SEARCH_ENGINE = os.getenv("SEARCH_ENGINE", "FaissFlat")
+DISTANCE_STRATEGY = os.getenv("DISTANCE_STRATEGY", "IP")
 
 logger = CustomLogger("opea_dataprep_vdms_multimodal")
 logflag = os.getenv("LOGFLAG", False)
@@ -72,6 +74,7 @@ def store_into_vectordb(self, vs, metadata_file_path, dimensions):
             metadata_list = [data]
             if vs.selected_db == "vdms":
                 vs.video_db.add_videos(
+                    texts=video_name_list,
                     paths=video_name_list,
                     metadatas=metadata_list,
                     start_time=[data["timestamp"]],
@@ -145,14 +148,21 @@ async def ingest_videos(self, files: List[UploadFile] = File(None)):
         # init meanclip model
         model = self.setup_vclip_model(meanclip_cfg, device="cpu")
         vs = store_embeddings.VideoVS(
-            host, port, selected_db, model, collection_name, embedding_dimensions=vector_dimensions
+            host,
+            port,
+            selected_db,
+            model,
+            collection_name,
+            embedding_dimensions=vector_dimensions,
+            engine=SEARCH_ENGINE,
+            distance_strategy=DISTANCE_STRATEGY,
         )
         logger.info("done creating DB, sleep 5s")
         await asyncio.sleep(5)
 
         self.generate_embeddings(config, vector_dimensions, vs)
 
-        return {"message": "Videos ingested successfully"}
+        return {"status": 200, "message": "Videos ingested successfully"}
 
     async def get_videos(self):
         """Returns list of names of uploaded videos saved on the server."""

comps/dataprep/src/requirements.txt

@@ -9,7 +9,7 @@ einops
 elasticsearch
 fastapi
 future
-graspologic 
+graspologic
 html2text
 huggingface_hub
 ipython
@@ -21,9 +21,10 @@ langchain-openai
 langchain-pinecone
 langchain-redis
 langchain-text-splitters
+langchain-vdms>=0.1.4
 langchain_huggingface
 langchain_milvus
-llama-index 
+llama-index
 llama-index-core==0.12.19
 llama-index-embeddings-text-embeddings-inference
 llama-index-graph-stores-neo4j
@@ -37,11 +38,15 @@ openai
 openai-whisper
 opencv-python
 opensearch-py
+opentelemetry-api==1.27.0
+opentelemetry-exporter-otlp==1.27.0
+opentelemetry-sdk==1.27.0
 pandas
 pgvector==0.2.5
 Pillow
 pinecone-client
 prometheus-fastapi-instrumentator
+protobuf==4.24.2
 psycopg2
 pymupdf
 pyspark
@@ -60,5 +65,4 @@ typing
 tzlocal
 unstructured[all-docs]
 uvicorn
-vdms
 webvtt-py

comps/retrievers/deployment/docker_compose/compose.yaml

@@ -196,6 +196,18 @@ services:
       tei-embedding-serving:
         condition: service_healthy
 
+  retriever-vdms-multimodal:
+    extends: retriever
+    container_name: retriever-vdms-multimodal
+    environment:
+      RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_VDMS"
+      VDMS_INDEX_NAME: ${INDEX_NAME}
+      VDMS_HOST: ${host_ip}
+      VDMS_PORT: ${VDMS_PORT}
+      VDMS_USE_CLIP: ${VDMS_USE_CLIP}
+    depends_on:
+      vdms-vector-db:
+        condition: service_healthy
 
 networks:
   default:

comps/retrievers/src/Dockerfile

@@ -25,8 +25,7 @@ RUN pip install --no-cache-dir --upgrade pip setuptools && \
         PIP_EXTRA_INDEX_URL=""; \
     fi && \
     pip install --no-cache-dir torch torchvision ${PIP_EXTRA_INDEX_URL} && \
-    pip install --no-cache-dir ${PIP_EXTRA_INDEX_URL} -r /home/user/comps/retrievers/src/requirements.txt && \
-    pip install opentelemetry-api==1.29.0 opentelemetry-exporter-otlp==1.29.0 opentelemetry-sdk==1.29.0
+    pip install --no-cache-dir ${PIP_EXTRA_INDEX_URL} -r /home/user/comps/retrievers/src/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 

comps/retrievers/src/integrations/config.py

@@ -184,5 +184,5 @@ def format_opensearch_conn_from_env():
 VDMS_PORT = int(os.getenv("VDMS_PORT", 55555))
 VDMS_INDEX_NAME = os.getenv("VDMS_INDEX_NAME", "rag_vdms")
 VDMS_USE_CLIP = int(os.getenv("VDMS_USE_CLIP", 0))
-SEARCH_ENGINE = "FaissFlat"
-DISTANCE_STRATEGY = "IP"
+SEARCH_ENGINE = os.getenv("SEARCH_ENGINE", "FaissFlat")
+DISTANCE_STRATEGY = os.getenv("DISTANCE_STRATEGY", "IP")