feat: add supabase_vectorstore (#117)

* 魔改一下 SupabaseVectorStore，支持 doc 表字段扩展

![image](https://github.com/ant-xuexiao/petercat/assets/10885578/d832aed9-62a3-475e-a2cb-3c11fe4c66c5)

Author: RaoHai

server/agent/base.py

@@ -8,10 +8,10 @@
 )
 from langchain_core.messages import AIMessage, FunctionMessage, HumanMessage
 from langchain.agents.output_parsers.openai_tools import OpenAIToolsAgentOutputParser
-from langchain.prompts import PromptTemplate, MessagesPlaceholder
+from langchain.prompts import MessagesPlaceholder
 from langchain_core.utils.function_calling import convert_to_openai_tool
 from langchain_core.prompts import ChatPromptTemplate
-from langchain.utilities.tavily_search import TavilySearchAPIWrapper
+from langchain_community.utilities.tavily_search import TavilySearchAPIWrapper
 from langchain.tools.tavily_search import TavilySearchResults
 from langchain_openai import ChatOpenAI
 from uilts.env import get_env_variable

server/data_class.py

@@ -39,6 +39,8 @@ class GitIssueConfig(BaseModel):
 class GitDocConfig(BaseModel):
     repo_name: str
     file_path: str
+    commit_id: Optional[str] = None
+    commit_sha: Optional[str] = None
     """File path of the documentation file. eg:'docs/blog/build-ghost.zh-CN.md'"""
     branch: Optional[str] = 'main'
 

server/rag/retrieval.py

@@ -1,6 +1,7 @@
 import json
+from typing import Any, Optional
 from langchain_openai import OpenAIEmbeddings
-from langchain_community.vectorstores import SupabaseVectorStore
+from rag.supabase_vectorstore import SupabaseVectorStore
 from db.supabase.client import get_client
 from data_class import GitDocConfig, GitIssueConfig, S3Config
 from rag.github_file_loader import GithubFileLoader
@@ -23,15 +24,15 @@ def convert_document_to_dict(document):
 
 def init_retriever():
     embeddings = OpenAIEmbeddings()
-    db = SupabaseVectorStore(
+    vector_store = SupabaseVectorStore(
       embedding=embeddings,
       client=get_client(),
       table_name=TABLE_NAME,
       query_name=QUERY_NAME,
       chunk_size=CHUNK_SIZE,
     )
 
-    return db.as_retriever()
+    return vector_store.as_retriever()
 
 
 def init_s3_Loader(config: S3Config):
@@ -61,38 +62,43 @@ def init_github_file_loader(config: GitDocConfig):
     )
     return loader
 
-def supabase_embedding(documents):
+def supabase_embedding(documents, **kwargs: Any):
     from langchain_text_splitters import CharacterTextSplitter
 
     try:    
         text_splitter = CharacterTextSplitter(chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP)
         docs = text_splitter.split_documents(documents)
         embeddings = OpenAIEmbeddings()
-        SupabaseVectorStore.from_documents(
+        vector_store = SupabaseVectorStore.from_documents(
             docs,
             embeddings,
             client=get_client(),
             table_name=TABLE_NAME,
             query_name=QUERY_NAME,
             chunk_size=CHUNK_SIZE,
+            **kwargs
         )
-        return json.dumps({
-            "success": True,
-            "message": "Knowledge added successfully!",
-            "docs_len": len(documents)
-        }) 
+        return vector_store
     except Exception as e:
-        return json.dumps({
-            "success": False,
-            "message": str(e)
-        })
+        print(e)
+        return None
 
 
-def add_knowledge_by_issues(config: GitIssueConfig):    
+def add_knowledge_by_issues(config: GitIssueConfig, ):    
     try:
         loader = init_github_issue_loader(config)
         documents = loader.load()
-        supabase_embedding(documents)
+        store = supabase_embedding(documents, repo_name=config.repo_name)
+        if(store):
+            return json.dumps({
+                "success": True,
+                "message": "Knowledge added successfully!",
+            }) 
+        else:
+            return json.dumps({
+                "success": False,
+                "message": "Knowledge not added!"
+            })
     except Exception as e:
         return json.dumps({
             "success": False,
@@ -103,7 +109,18 @@ def add_knowledge_by_doc(config: GitDocConfig):
     try:
         loader = init_github_file_loader(config)
         documents = loader.load()
-        supabase_embedding(documents)
+        store = supabase_embedding(documents, repo_name=config.repo_name, commit_id=config.commit_id, commit_sha=config.commit_sha)
+        if(store):
+            return json.dumps({
+                "success": True,
+                "message": "Knowledge added successfully!",
+            }) 
+        else:
+            return json.dumps({
+                "success": False,
+                "message": "Knowledge not added!"
+            })
+
     except Exception as e:
         return json.dumps({
             "success": False,