Merge branch 'main' into release-1.0

Author: shaohuzhang1

.github/workflows/build-and-push.yml

@@ -48,7 +48,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: main
+          ref: ${{ github.ref_name }}
       - name: Prepare
         id: prepare
         run: |
@@ -104,7 +104,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: main
+          ref: ${{ github.ref_name }}
       - name: Prepare
         id: prepare
         run: |

.github/workflows/typos_check.yml

@@ -0,0 +1,13 @@
+name: Typos Check
+on: [push, pull_request]
+
+jobs:
+  run:
+    name: Spell Check with Typos
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Actions Repository
+      uses: actions/checkout@v2
+
+    - name: Check spelling
+      uses: crate-ci/typos@master

.gitignore

@@ -42,7 +42,7 @@ share/python-wheels/
 MANIFEST
 
 # PyInstaller
-#  Usually these files are written by a python script froms a template
+#  Usually these files are written by a python script forms a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec

README.md

@@ -51,14 +51,22 @@ docker run -d --name=maxkb -p 8080:8080 -v ~/.maxkb:/var/lib/postgresql/data 1pa
 
 -   前端：[Vue.js](https://cn.vuejs.org/)
 -   后端：[Python / Django](https://www.djangoproject.com/)
--   Langchain：[Langchain](https://www.langchain.com/)
+-   LangChain：[LangChain](https://www.langchain.com/)
 -   向量数据库：[PostgreSQL / pgvector](https://www.postgresql.org/)
 -   大模型：Azure OpenAI、百度千帆大模型、[Ollama](https://github.com/ollama/ollama)
 
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=1Panel-dev/MaxKB&type=Date)](https://star-history.com/#1Panel-dev/MaxKB&Date)
 
+## 我们的其他开源产品
+
+- [JumpServer](https://github.com/jumpserver/jumpserver/) - 广受欢迎的开源堡垒机
+- [DataEase](https://github.com/dataease/dataease/) - 人人可用的开源数据可视化分析工具
+- [MeterSphere](https://github.com/metersphere/metersphere/) - 一站式开源自动化测试平台
+- [1Panel](https://github.com/1panel-dev/1panel/) - 现代化、开源的 Linux 服务器运维管理面板
+- [Halo](https://github.com/halo-dev/halo/) - 强大易用的开源建站工具
+
 ## License
 
 Copyright (c) 2014-2024 飞致云 FIT2CLOUD, All rights reserved.

apps/application/serializers/application_serializers.py

@@ -196,7 +196,7 @@ class Authentication(serializers.Serializer):
         access_token = serializers.CharField(required=True, error_messages=ErrMessage.char("access_token"))
 
         def auth(self, request, with_valid=True):
-            token = request.META.get('HTTP_AUTHORIZATION', None)
+            token = request.META.get('HTTP_AUTHORIZATION')
             token_details = None
             try:
                 # 校验token
@@ -257,7 +257,7 @@ def insert(self, application: Dict):
             application_model = ApplicationSerializer.Create.to_application_model(user_id, application)
             dataset_id_list = application.get('dataset_id_list', [])
             application_dataset_mapping_model_list = [
-                ApplicationSerializer.Create.to_application_dateset_mapping(application_model.id, dataset_id) for
+                ApplicationSerializer.Create.to_application_dataset_mapping(application_model.id, dataset_id) for
                 dataset_id in dataset_id_list]
             # 插入应用
             application_model.save()
@@ -280,7 +280,7 @@ def to_application_model(user_id: str, application: Dict):
                                )
 
         @staticmethod
-        def to_application_dateset_mapping(application_id: str, dataset_id: str):
+        def to_application_dataset_mapping(application_id: str, dataset_id: str):
             return ApplicationDatasetMapping(id=uuid.uuid1(), application_id=application_id, dataset_id=dataset_id)
 
     class HitTest(serializers.Serializer):

apps/application/serializers/chat_message_serializers.py

@@ -178,12 +178,12 @@ def chat(self):
         client_id = self.data.get('client_id')
         client_type = self.data.get('client_type')
         chat_info = self.is_valid(raise_exception=True)
-        pipline_manage_builder = PipelineManage.builder()
+        pipeline_manage_builder = PipelineManage.builder()
         # 如果开启了问题优化,则添加上问题优化步骤
         if chat_info.application.problem_optimization:
-            pipline_manage_builder.append_step(BaseResetProblemStep)
+            pipeline_manage_builder.append_step(BaseResetProblemStep)
         # 构建流水线管理器
-        pipline_message = (pipline_manage_builder.append_step(BaseSearchDatasetStep)
+        pipeline_message = (pipeline_manage_builder.append_step(BaseSearchDatasetStep)
                            .append_step(BaseGenerateHumanMessageStep)
                            .append_step(BaseChatStep)
                            .build())
@@ -198,8 +198,8 @@ def chat(self):
         params = chat_info.to_pipeline_manage_params(message, get_post_handler(chat_info), exclude_paragraph_id_list,
                                                      client_id, client_type, stream)
         # 运行流水线作业
-        pipline_message.run(params)
-        return pipline_message.context['chat_result']
+        pipeline_message.run(params)
+        return pipeline_message.context['chat_result']
 
     @staticmethod
     def re_open_chat(chat_id: str):

apps/application/template/embed.js

@@ -247,7 +247,7 @@ function initMaxkbStyle(root){
     #maxkb #maxkb-chat-container{
         z-index:10000;position: relative;
               border-radius: 8px;
-              border: 1px solid var(--N300, #DEE0E3);
+              border: 1px solid #ffffff;
               background: linear-gradient(188deg, rgba(235, 241, 255, 0.20) 39.6%, rgba(231, 249, 255, 0.20) 94.3%), #EFF0F1;
               box-shadow: 0px 4px 8px 0px rgba(31, 35, 41, 0.10);
               position: fixed;bottom: 20px;right: 45px;overflow: hidden;

apps/common/auth/authenticate.py

@@ -47,8 +47,7 @@ def get_token_details(self):
 class TokenAuth(TokenAuthentication):
     # 重新 authenticate 方法，自定义认证规则
     def authenticate(self, request):
-        auth = request.META.get('HTTP_AUTHORIZATION', None
-                                )
+        auth = request.META.get('HTTP_AUTHORIZATION')
         # 未认证
         if auth is None:
             raise AppAuthenticationFailed(1003, '未登录,请先登录')

apps/common/event/listener_manage.py

@@ -215,7 +215,7 @@ def delete_embedding_by_dataset_id_list(source_ids: List[str]):
 
     @staticmethod
     @poxy
-    def init_embedding_model(ags):
+    def init_embedding_model(ages):
         EmbeddingModel.get_embedding_model()
 
     def run(self):

apps/common/froms/__init__.py renamed to apps/common/forms/__init__.py

@@ -8,7 +8,7 @@
 """
 from typing import Dict
 
-from common.froms.base_field import BaseExecField, TriggerType
+from common.forms.base_field import BaseExecField, TriggerType
 
 
 class ArrayCard(BaseExecField):

apps/common/froms/array_object_card.py renamed to apps/common/forms/array_object_card.py

@@ -6,7 +6,7 @@
     @date：2023/11/1 16:04
     @desc:
 """
-from common.froms import BaseField
+from common.forms import BaseField
 
 
 class BaseForm:

apps/common/froms/base_field.py renamed to apps/common/forms/base_field.py

@@ -8,7 +8,7 @@
 """
 from typing import List, Dict
 
-from common.froms.base_field import BaseExecField, TriggerType
+from common.forms.base_field import BaseExecField, TriggerType
 
 
 class MultiSelect(BaseExecField):

apps/common/froms/base_form.py renamed to apps/common/forms/base_form.py

@@ -8,7 +8,7 @@
 """
 from typing import Dict
 
-from common.froms.base_field import BaseExecField, TriggerType
+from common.forms.base_field import BaseExecField, TriggerType
 
 
 class ObjectCard(BaseExecField):

apps/common/froms/multi_select.py renamed to apps/common/forms/multi_select.py

@@ -8,7 +8,7 @@
 """
 from typing import Dict
 
-from common.froms import BaseField, TriggerType
+from common.forms import BaseField, TriggerType
 
 
 class PasswordInputField(BaseField):

apps/common/froms/object_card.py renamed to apps/common/forms/object_card.py

@@ -8,7 +8,7 @@
 """
 from typing import List, Dict
 
-from common.froms.base_field import BaseExecField, TriggerType
+from common.forms.base_field import BaseExecField, TriggerType
 
 
 class Radio(BaseExecField):

apps/common/froms/password_input.py renamed to apps/common/forms/password_input.py

@@ -8,7 +8,7 @@
 """
 from typing import List, Dict
 
-from common.froms.base_field import BaseExecField, TriggerType
+from common.forms.base_field import BaseExecField, TriggerType
 
 
 class Radio(BaseExecField):

apps/common/froms/radio_button_field.py renamed to apps/common/forms/radio_button_field.py

@@ -8,7 +8,7 @@
 """
 from typing import List, Dict
 
-from common.froms.base_field import BaseExecField, TriggerType
+from common.forms.base_field import BaseExecField, TriggerType
 
 
 class Radio(BaseExecField):

apps/common/froms/radio_card_field.py renamed to apps/common/forms/radio_card_field.py

@@ -8,7 +8,7 @@
 """
 from typing import List, Dict
 
-from common.froms.base_field import TriggerType, BaseExecField
+from common.forms.base_field import TriggerType, BaseExecField
 
 
 class SingleSelect(BaseExecField):

apps/common/froms/radio_field.py renamed to apps/common/forms/radio_field.py

@@ -8,7 +8,7 @@
 """
 from typing import Dict
 
-from common.froms.base_field import BaseExecField, TriggerType
+from common.forms.base_field import BaseExecField, TriggerType
 
 
 class TabCard(BaseExecField):

apps/common/froms/single_select_field.py renamed to apps/common/forms/single_select_field.py

@@ -8,7 +8,7 @@
 """
 from typing import Dict
 
-from common.froms.base_field import TriggerType, BaseExecField
+from common.forms.base_field import TriggerType, BaseExecField
 
 
 class TableRadio(BaseExecField):

apps/common/froms/tab_card.py renamed to apps/common/forms/tab_card.py

@@ -8,7 +8,7 @@
 """
 from typing import Dict
 
-from common.froms.base_field import TriggerType, BaseExecField
+from common.forms.base_field import TriggerType, BaseExecField
 
 
 class TableRadio(BaseExecField):

apps/common/froms/table_checkbox.py renamed to apps/common/forms/table_checkbox.py

@@ -8,7 +8,7 @@
 """
 from typing import Dict
 
-from common.froms.base_field import BaseField, TriggerType
+from common.forms.base_field import BaseField, TriggerType
 
 
 class TextInputField(BaseField):

apps/common/froms/table_radio.py renamed to apps/common/forms/table_radio.py

@@ -9,8 +9,6 @@
 
 
 def get_file_content(path):
-    file = open(path, "r",
-                encoding='utf-8')
-    content = file.read()
-    file.close()
+    with open(path, "r", encoding='utf-8') as file:
+        content = file.read()
     return content

apps/common/froms/text_input_field.py renamed to apps/common/forms/text_input_field.py

@@ -336,6 +336,7 @@ def parse(self, text: str):
         :return: 解析后数据 {content:段落数据,keywords:[‘段落关键词’],parent_chain:['段落父级链路']}
         """
         text = text.replace('\r', '\n')
+        text = text.replace("\0", '')
         result_tree = self.parse_to_tree(text, 0)
         result = result_tree_to_paragraph(result_tree, [], [])
         return [item for item in [self.post_reset_paragraph(row) for row in result] if

apps/common/util/file_util.py

@@ -11,11 +11,15 @@
 from setting.models_provider.impl.azure_model_provider.azure_model_provider import AzureModelProvider
 from setting.models_provider.impl.ollama_model_provider.ollama_model_provider import OllamaModelProvider
 from setting.models_provider.impl.openai_model_provider.openai_model_provider import OpenAIModelProvider
+from setting.models_provider.impl.qwen_model_provider.qwen_model_provider import QwenModelProvider
 from setting.models_provider.impl.wenxin_model_provider.wenxin_model_provider import WenxinModelProvider
+from setting.models_provider.impl.kimi_model_provider.kimi_model_provider import KimiModelProvider
 
 
 class ModelProvideConstants(Enum):
     model_azure_provider = AzureModelProvider()
     model_wenxin_provider = WenxinModelProvider()
     model_ollama_provider = OllamaModelProvider()
     model_openai_provider = OpenAIModelProvider()
+    model_kimi_provider = KimiModelProvider()
+    model_qwen_provider = QwenModelProvider()

apps/common/util/split_model.py

@@ -12,9 +12,9 @@
 from langchain.schema import HumanMessage
 from langchain_community.chat_models.azure_openai import AzureChatOpenAI
 
-from common import froms
+from common import forms
 from common.exception.app_exception import AppApiException
-from common.froms import BaseForm
+from common.forms import BaseForm
 from common.util.file_util import get_file_content
 from setting.models_provider.base_model_provider import IModelProvider, ModelProvideInfo, BaseModelCredential, \
     ModelInfo, \
@@ -51,11 +51,11 @@ def is_valid(self, model_type: str, model_name, model_credential: Dict[str, obje
     def encryption_dict(self, model: Dict[str, object]):
         return {**model, 'api_key': super().encryption(model.get('api_key', ''))}
 
-    api_base = froms.TextInputField('API 域名', required=True)
+    api_base = forms.TextInputField('API 域名', required=True)
 
-    api_key = froms.PasswordInputField("API Key", required=True)
+    api_key = forms.PasswordInputField("API Key", required=True)
 
-    deployment_name = froms.TextInputField("部署名", required=True)
+    deployment_name = forms.TextInputField("部署名", required=True)
 
 
 class DefaultAzureLLMModelCredential(BaseForm, BaseModelCredential):
@@ -87,13 +87,13 @@ def is_valid(self, model_type: str, model_name, model_credential: Dict[str, obje
     def encryption_dict(self, model: Dict[str, object]):
         return {**model, 'api_key': super().encryption(model.get('api_key', ''))}
 
-    api_version = froms.TextInputField("api_version", required=True)
+    api_version = forms.TextInputField("api_version", required=True)
 
-    api_base = froms.TextInputField('API 域名', required=True)
+    api_base = forms.TextInputField('API 域名', required=True)
 
-    api_key = froms.PasswordInputField("API Key", required=True)
+    api_key = forms.PasswordInputField("API Key", required=True)
 
-    deployment_name = froms.TextInputField("部署名", required=True)
+    deployment_name = forms.TextInputField("部署名", required=True)
 
 
 azure_llm_model_credential = AzureLLMModelCredential()

apps/setting/models_provider/constants/model_provider_constants.py

@@ -0,0 +1,8 @@
+# coding=utf-8
+"""
+    @project: maxkb
+    @Author：虎
+    @file： __init__.py.py
+    @date：2023/10/31 17:16
+    @desc:
+"""