CICD github action build publish pypi、Release Tag (chatchat-space#4260)

## Add CI/CD build process, which includes the following tasks: compiling and building, releasing to test PyPI, testing with pytest, releasing to PyPI, and publishing a Release Tag.

### your configuration env to github  Repository secrets

[using-secrets-in-github-actions](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions)

### pypi 
pypi token generate for docs [publishers](https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/) or [publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/)
#### Environment 
- PYPI_API_TOKEN
- TEST_PYPI_API_TOKEN

 

### Release Tag
 
[release-action](https://github.com/ncipollo/release-action)

[managing-your-personal-access-tokens](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens)


增加CICD构建流程，包含了如下作业，编译构建、发布test pypi、测试pytest、发布pypi、发布Release Tag

需要配置仓库变量才能使用，可以在这里查看配置方法
[using-secrets-in-github-actions](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions)

### pypi 
pypi token生成可以选择配置变量也可以选择pipy加入开发人员[publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/)
#### Environment 
- PYPI_API_TOKEN
- TEST_PYPI_API_TOKEN

.github/workflows/_release.yml

@@ -2,21 +2,17 @@ name: release
 run-name: Release ${{ inputs.working-directory }} by @${{ github.actor }}
 on:
   workflow_call:
-    branches:
-      - dev
     inputs:
       working-directory:
         required: true
         type: string
         description: "From which folder this pipeline executes"
   workflow_dispatch:
-    branches:
-      - dev
     inputs:
       working-directory:
         required: true
         type: string
-        default: './chatchat-server'
+        default: './libs/model-providers'
         description: "From which folder this pipeline executes"
 env:
   PYTHON_VERSION: "3.8"
@@ -179,13 +175,7 @@ jobs:
       - pre-release-checks
     environment: Scheduled testing
     runs-on: ubuntu-latest
-    permissions:
-      # This permission is used for trusted publishing:
-      # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
-      #
-      # Trusted publishing has to also be configured on PyPI for each package:
-      # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
-      id-token: write
+
 
     defaults:
       run:
@@ -213,11 +203,12 @@ jobs:
           packages-dir: ${{ inputs.working-directory }}/dist/
           verbose: true
           print-hash: true
+          user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
           # We overwrite any existing distributions with the same name and version.
           # This is *only for CI use* and is *extremely dangerous* otherwise!
           # https://github.com/pypa/gh-action-pypi-publish#tolerating-release-package-file-duplicates
-          skip-existing: true
+#          skip-existing: true
 
   mark-release:
     needs:
@@ -231,6 +222,7 @@ jobs:
       # This permission is needed by `ncipollo/release-action` to
       # create the GitHub release.
       contents: write
+      id-token: none
 
     defaults:
       run:

.github/workflows/_test_release.yml

@@ -2,8 +2,6 @@ name: test-release
 
 on:
   workflow_call:
-    branches:
-      - dev
     inputs:
       working-directory:
         required: true
@@ -67,13 +65,7 @@ jobs:
     needs:
       - build
     runs-on: ubuntu-latest
-    permissions:
-      # This permission is used for trusted publishing:
-      # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
-      #
-      # Trusted publishing has to also be configured on PyPI for each package:
-      # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
-      id-token: write
+    environment: Scheduled test pypi publish
 
     steps:
       - uses: actions/checkout@v4
@@ -86,12 +78,13 @@ jobs:
       - name: Publish to test PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
+          user: __token__
+          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           packages-dir: ${{ inputs.working-directory }}/dist/
           verbose: true
           print-hash: true
           repository-url: https://test.pypi.org/legacy/
-          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           # We overwrite any existing distributions with the same name and version.
           # This is *only for CI use* and is *extremely dangerous* otherwise!
           # https://github.com/pypa/gh-action-pypi-publish#tolerating-release-package-file-duplicates
-          skip-existing: true
+#          skip-existing: true

libs/chatchat-server/Makefile

@@ -0,0 +1,85 @@
+.PHONY: all format lint test tests test_watch integration_tests docker_tests help extended_tests
+
+# Default target executed when no arguments are given to make.
+all: help
+
+######################
+# TESTING AND COVERAGE
+######################
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/unit_tests/
+
+# Run unit tests and generate a coverage report.
+coverage:
+	poetry run pytest --cov \
+		--cov-config=.coveragerc \
+		--cov-report xml \
+		--cov-report term-missing:skip-covered \
+		$(TEST_FILE)
+
+test tests:
+	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
+
+extended_tests:
+	poetry run pytest --disable-socket --allow-unix-socket --only-extended tests/unit_tests
+
+test_watch:
+	poetry run ptw --snapshot-update --now . -- -x --disable-socket --allow-unix-socket tests/unit_tests
+
+test_watch_extended:
+	poetry run ptw --snapshot-update --now . -- -x --disable-socket --allow-unix-socket --only-extended tests/unit_tests
+
+integration_tests:
+	poetry run pytest tests/integration_tests
+
+scheduled_tests:
+	poetry run pytest -m scheduled tests/integration_tests
+
+
+######################
+# LINTING AND FORMATTING
+######################
+
+# Define a variable for Python and notebook files.
+PYTHON_FILES=.
+MYPY_CACHE=.mypy_cache
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=libs/langchain --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+lint_package: PYTHON_FILES=chatchat
+lint_tests: PYTHON_FILES=tests
+lint_tests: MYPY_CACHE=.mypy_cache_test
+
+lint lint_diff lint_package lint_tests:
+	./scripts/check_pydantic.sh .
+	./scripts/lint_imports.sh
+	poetry run ruff .
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES) --diff
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || mkdir -p $(MYPY_CACHE) && poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE)
+
+format format_diff:
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+######################
+# HELP
+######################
+
+help:
+	@echo '-- LINTING --'
+	@echo 'format                       - run code formatters'
+	@echo 'lint                         - run linters'
+	@echo 'spell_check               	- run codespell on the project'
+	@echo 'spell_fix               		- run codespell on the project and fix the errors'
+	@echo '-- TESTS --'
+	@echo 'coverage                     - run unit tests and generate coverage report'
+	@echo 'test                         - run unit tests'
+	@echo 'tests                        - run unit tests (alias for "make test")'
+	@echo 'test TEST_FILE=<test_file>   - run all tests in file'

libs/chatchat-server/chatchat/server/utils.py

@@ -24,7 +24,7 @@
     Tuple,
     Literal,
 )
-
+import socket
 from chatchat.configs import (log_verbose, HTTPX_DEFAULT_TIMEOUT,
                               DEFAULT_LLM_MODEL, DEFAULT_EMBEDDING_MODEL, TEMPERATURE,
                               MODEL_PLATFORMS)
@@ -760,3 +760,9 @@ def get_tool_config(name: str = None) -> Dict:
         return TOOL_CONFIG
     else:
         return TOOL_CONFIG.get(name, {})
+
+
+
+def is_port_in_use(port):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        return sock.connect_ex(('localhost', port)) == 0

libs/chatchat-server/chatchat/startup.py

@@ -37,7 +37,6 @@ async def lifespan(app: FastAPI):
 def run_init_server(
         model_platforms_shard: Dict,
         started_event: mp.Event = None,
-        run_mode: str = None,
         model_providers_cfg_path: str = None,
         provider_host: str = None,
         provider_port: int = None):
@@ -320,8 +319,7 @@ def process_count():
         process = Process(
             target=run_init_server,
             name=f"Model providers Server",
-            kwargs=dict(model_platforms_shard=model_platforms_shard, started_event=model_providers_started,
-                        run_mode=run_mode),
+            kwargs=dict(model_platforms_shard=model_platforms_shard, started_event=model_providers_started),
             daemon=True,
         )
         processes["model_providers"] = process

libs/chatchat-server/langchain_chatchat/__init__.py

@@ -0,0 +1,35 @@
+import importlib
+import sys
+import types
+
+# 动态导入 a_chatchat 模块
+chatchat = importlib.import_module('chatchat')
+
+# 创建新的模块对象
+module = types.ModuleType('langchain_chatchat')
+sys.modules['langchain_chatchat'] = module
+
+# 把 a_chatchat 的所有属性复制到 langchain_chatchat
+for attr in dir(chatchat):
+    if not attr.startswith('_'):
+        setattr(module, attr, getattr(chatchat, attr))
+
+
+# 动态导入子模块
+def import_submodule(name):
+    full_name = f'chatchat.{name}'
+    submodule = importlib.import_module(full_name)
+    sys.modules[f'langchain_chatchat.{name}'] = submodule
+    for attr in dir(submodule):
+        if not attr.startswith('_'):
+            setattr(module, attr, getattr(submodule, attr))
+
+
+# 需要的子模块列表，自己添加
+submodules = ['configs', 'server',
+              'startup', 'webui_pages'
+              ]
+
+# 导入所有子模块
+for submodule in submodules:
+    import_submodule(submodule)

libs/chatchat-server/pyproject.toml

@@ -1,11 +1,12 @@
 [tool.poetry]
 name = "langchain-chatchat"
-version = "0.3.0.20240619"
+version = "0.3.0.20240621.3"
 description = ""
 authors = ["chatchat"]
 readme = "README.md"
 packages = [
-    {include = "chatchat"}
+    {include = "chatchat"},
+    {include = "langchain_chatchat"}
 ]
 
 [tool.poetry.scripts]
@@ -66,7 +67,6 @@ pymysql = "^1.1.0"
 [tool.poetry.extras]
 xinference = ["xinference_client"]
 zhipuai = ["zhipuai"]
-cli = ["typer"]
 
 # An extra used to be able to add extended testing.
 # Please use new-line on formatting to make it easier to add new packages without
@@ -106,13 +106,11 @@ extended_testing = [
     "scikit-learn",
     "streamlit",
     "pyspark",
-    "openai",
     "sympy",
     "rapidfuzz",
     "jsonschema",
     "rank-bm25",
     "geopandas",
-    "jinja2",
     "gitpython",
     "newspaper3k",
     "nvidia-riva-client",
@@ -141,8 +139,6 @@ extended_testing = [
     "tree-sitter-languages",
     "azure-ai-documentintelligence",
     "oracle-ads",
-    "zhipuai",
-    "httpx",
     "elasticsearch",
     "hdbcli",
     "oci",
@@ -263,7 +259,7 @@ dotenv = "dotenv:plugin"
 
 
 # https://python-poetry.org/docs/repositories/
-[[tool.poetry.source]]
-name = "tsinghua"
-url = "https://pypi.tuna.tsinghua.edu.cn/simple/"
-priority = "primary"
+#[[tool.poetry.source]]
+#name = "tsinghua"
+#url = "https://pypi.tuna.tsinghua.edu.cn/simple/"
+#priority = "primary"

libs/chatchat-server/scripts/check_imports.py

@@ -0,0 +1,22 @@
+import random
+import string
+import sys
+import traceback
+from importlib.machinery import SourceFileLoader
+
+if __name__ == "__main__":
+    files = sys.argv[1:]
+    has_failure = False
+    for file in files:
+        try:
+            module_name = "".join(
+                random.choice(string.ascii_letters) for _ in range(20)
+            )
+            SourceFileLoader(module_name, file).load_module()
+        except Exception:
+            has_failure = True
+            print(file)  # noqa: T201
+            traceback.print_exc()
+            print()  # noqa: T201
+
+    sys.exit(1 if has_failure else 0)

libs/chatchat-server/scripts/check_pydantic.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# This script searches for lines starting with "import pydantic" or "from pydantic"
+# in tracked files within a Git repository.
+#
+# Usage: ./scripts/check_pydantic.sh /path/to/repository
+
+# Check if a path argument is provided
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 /path/to/repository"
+  exit 1
+fi
+
+repository_path="$1"
+
+# Search for lines matching the pattern within the specified repository
+result=$(git -C "$repository_path" grep -E '^import pydantic|^from pydantic')
+
+# Check if any matching lines were found
+if [ -n "$result" ]; then
+  echo "ERROR: The following lines need to be updated:"
+  echo "$result"
+  echo "Please replace the code with an import from langchain_core.pydantic_v1."
+  echo "For example, replace 'from pydantic import BaseModel'"
+  echo "with 'from langchain_core.pydantic_v1 import BaseModel'"
+  exit 1
+fi

libs/chatchat-server/scripts/lint_imports.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eu
+
+# Initialize a variable to keep track of errors
+errors=0
+
+# make sure not importing from chatchat
+git --no-pager grep '^from chatchat\.' . && errors=$((errors+1))
+
+# Decide on an exit status based on the errors
+if [ "$errors" -gt 0 ]; then
+    exit 1
+else
+    exit 0
+fi