积累修复 (#4561)

docker update to 0.3.1.1 fix: duplicated logs in chatchat.log fix: nltk data path was not set correctlly fix: 重建知识库时只处理了一个知识库 --------- Co-authored-by: imClumsyPanda <[email protected]> Co-authored-by: yuehua <[email protected]> Co-authored-by: yuehuazhang <[email protected]>
chatchat-space · Jul 18, 2024 · cbfdd72 · cbfdd72
1 parent ea09c4c
commit cbfdd72
Show file tree

Hide file tree

Showing 11 changed files with 94 additions and 43 deletions.
diff --git a/README.md b/README.md
@@ -303,7 +303,9 @@ chatchat start -a
 ### Docker 部署
 
 ```shell
-docker pull chatimage/chatchat:0.3.0-2024-0624
+docker pull chatimage/chatchat:0.3.1.1-2024-0714
+
+docker pull ccr.ccs.tencentyun.com/chatchat/chatchat:0.3.1.1-2024-0714 # 国内镜像
 ```
 
 > [!important]

diff --git a/docker/.dockerignore b/docker/.dockerignore
@@ -16,4 +16,8 @@ requirements_lite.txt
 requirements_webui.txt
 # bge-large-zh-v1.5
 bge-large-zh-v1.5/README.md
-# chatglm3-6b
+# chatglm3-6b
+../docs/contributing/
+../docs/install/
+../README.md
+../README_en.md
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,28 +1,53 @@
-# Base Image
+# 基础镜像
 FROM python:3.11
-# Labels
-LABEL maintainer=chatchat
-# Environment Variables
-ENV HOME=/usr/local/lib/python3.11/site-packages/chatchat
-# Init Environment
+LABEL maintainer=Langchain-Chatchat
+WORKDIR /root
+# 环境变量
+ENV CHATCHAT_ROOT=/root/chatchat_data
+ENV RELEASE=0.3.1.1
+
+# 初始化环境
 RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
     echo "Asia/Shanghai" > /etc/timezone
-# Install Dependencies
 RUN apt-get update -y && \
+    apt-get install -y git && \
     apt-get install -y --no-install-recommends libgl1 libglib2.0-0 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
-RUN pip install openpyxl networkx faiss-cpu jq unstructured[pdf] \
-    opencv-python rapidocr-onnxruntime PyMuPDF rank_bm25 youtube_search python-docx
-# Install Chatchat
-RUN pip install --index-url https://pypi.python.org/simple/ langchain-chatchat -U
+RUN pip install --upgrade pip setuptools
+RUN #pip install --index-url https://pypi.python.org/simple/ pipx && pipx install poetry --force
+# Add poetry to PATH
+#ENV PATH="/root/.local/bin:${PATH}"
+
+# 下载 Langchain-Chatchat
+RUN #git clone https://github.com/chatchat-space/Langchain-Chatchat.git
+RUN pip install --index-url https://pypi.python.org/simple/ "langchain-chatchat[xinference]"==$RELEASE -U
+
+#WORKDIR /root/Langchain-Chatchat/libs/chatchat-server
+
+# 设置 poetry 官方源
+#RUN poetry config virtualenvs.create false && \
+#    poetry config repositories.pypi.url https://pypi.org/simple/
+
+# 安装依赖
+RUN #poetry install --with lint,test -E xinference
+
+# 初始化配置
+RUN #python cli.py init
+RUN chatchat init
+
 # Install ModelProvider
-RUN pip install xinference-client
-# Make Custom Settings
-RUN chatchat-config server --default_bind_host=0.0.0.0 && \
-    chatchat-config model --default_llm_model qwen2-instruct
-# Copy Data
-ADD /docker/data.tar.gz $HOME/
-WORKDIR $HOME
+#RUN pip install xinference-client
+
+# 将监听 IP 从 localhost 统一改为 0.0.0.0
+RUN sed -i 's/127.0.0.1/0.0.0.0/g' $CHATCHAT_ROOT/basic_settings.yaml
+
+# 初始化知识库文件
+ADD /docker/data.tar.gz $CHATCHAT_ROOT/
+
+WORKDIR $CHATCHAT_ROOT
+
 EXPOSE 7861 8501
-ENTRYPOINT ["chatchat", "-a"]
+
+#ENTRYPOINT ["python", "cli.py", "start", "-a"]
+ENTRYPOINT ["chatchat", "start", "-a"]
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
@@ -1,7 +1,7 @@
 version: '3.9'
 services:
   xinference:
-    image: xprobe/xinference:v0.12.1
+    image: xprobe/xinference:v0.12.3
     restart: always
     command: xinference-local -H 0.0.0.0
     # ports: # 不使用 host network 时可打开.
@@ -25,12 +25,13 @@ services:
     # environment:
     #   - XINFERENCE_MODEL_SRC=modelscope
   chatchat:
-    image: chatimage/chatchat:0.3.0-2024-0624
+    image: chatimage/chatchat:0.3.1.1-2024-0714
+    # image: ccr.ccs.tencentyun.com/chatchat/chatchat:0.3.1.1-2024-0714
     restart: always
     # ports: # 不使用 host network 时可打开.
     #   - "7861:7861"
     #   - "8501:8501"
     network_mode: "host"
-    # 将本地路径(~/chatchat/data)挂载到容器默认数据路径(/usr/local/lib/python3.11/site-packages/chatchat/data)中
+    # 将本地路径(~/chatchat)挂载到容器默认数据路径($CHATCHAT_ROOT)中
     # volumes:
-    #   - ~/chatchat/data:/usr/local/lib/python3.11/site-packages/chatchat/data
+    #   - /root/chatchat:/root/chatchat_data
diff --git a/docs/install/README_docker.md b/docs/install/README_docker.md
@@ -59,7 +59,7 @@ docker-compose up -d
 ```text
 WARN[0000] /root/docker-compose.yaml: `version` is obsolete 
 NAME                IMAGE                           COMMAND                  SERVICE      CREATED         STATUS         PORTS
-root-chatchat-1     chatimage/chatchat:0.3.0-0622   "chatchat -a"            chatchat     3 minutes ago   Up 3 minutes   
+root-chatchat-1     chatimage/chatchat:0.3.1.1-2024-0714   "chatchat -a"            chatchat     3 minutes ago   Up 3 minutes   
 root-xinference-1   xprobe/xinference:v0.12.1       "/opt/nvidia/nvidia_…"   xinference   3 minutes ago   Up 3 minutes
 ```
 ```shell
@@ -95,7 +95,7 @@ mkdir -p ~/chatchat
 ```yaml
   (上文 ...)
   chatchat:
-    image: chatimage/chatchat:0.3.0-0622
+    image: chatimage/chatchat:0.3.1.1-2024-0714
     (省略 ...)
     # 将本地路径(~/chatchat/data)挂载到容器默认数据路径(/usr/local/lib/python3.11/site-packages/chatchat/data)中
     # volumes:
@@ -106,7 +106,7 @@ mkdir -p ~/chatchat
 ```yaml
   (上文 ...)
   chatchat:
-    image: chatimage/chatchat:0.3.0-0622
+    image: chatimage/chatchat:0.3.1.1-2024-0714
     (省略 ...)
     # 将本地路径(~/chatchat/data)挂载到容器默认数据路径(/usr/local/lib/python3.11/site-packages/chatchat/data)中
     volumes:
@@ -163,7 +163,7 @@ WARN[0000] /root/docker-compose.yaml: `version` is obsolete
 [root@VM-2-15-centos ~]# docker-compose ps
 WARN[0000] /root/docker-compose.yaml: `version` is obsolete 
 NAME                IMAGE                           COMMAND                  SERVICE      CREATED          STATUS          PORTS
-root-chatchat-1     chatimage/chatchat:0.3.0-0622   "chatchat -a"            chatchat     33 seconds ago   Up 32 seconds   
+root-chatchat-1     chatimage/chatchat:0.3.1.1-2024-0714   "chatchat -a"            chatchat     33 seconds ago   Up 32 seconds   
 root-xinference-1   xprobe/xinference:v0.12.1       "/opt/nvidia/nvidia_…"   xinference   45 minutes ago   Up 45 minutes   
 [root@VM-2-15-centos ~]# ss -anptl | grep -E '(8501|7861|9997)'
 LISTEN 0      128          0.0.0.0:9997       0.0.0.0:*    users:(("pt_main_thread",pid=1489804,fd=21))

diff --git a/libs/chatchat-server/README.md b/libs/chatchat-server/README.md
@@ -23,11 +23,16 @@ pip install langchain-chatchat
 
 # or if you use xinference to provide model API:
 # pip install langchain-chatchat[xinference]
+
+# if you update from an old version, we suggest to run init again to update yaml templates:
+# pip install -U langchain-chatchat
+# chatchat init
 ```
 
 详见这里的[安装指引](https://github.com/chatchat-space/Langchain-Chatchat/tree/master?tab=readme-ov-file#%E5%BF%AB%E9%80%9F%E4%B8%8A%E6%89%8B)。
 
 > 注意：chatchat请放在独立的虚拟环境中，比如conda，venv，virtualenv等
+> 
 > 已知问题，不能跟xinference一起安装，会让一些插件出bug，例如文件无法上传
 
 2. 源码安装
@@ -38,7 +43,9 @@ pip install langchain-chatchat
 3. Docker
 
 ```shell
-docker pull chatimage/chatchat:0.3.0-2024-0624
+docker pull chatimage/chatchat:0.3.1.1-2024-0714
+
+docker pull ccr.ccs.tencentyun.com/chatchat/chatchat:0.3.1.1-2024-0714 # 国内镜像
 ```
 
 > [!important]
@@ -68,10 +75,12 @@ chatchat init
 确保所有配置正确后（特别是 LLM 和 Embedding Model），执行下列命令创建默认知识库、启动服务：
 ```shell
 chatchat kb -r
-chatchat -a
+chatchat start -a
 ```
 如无错误将自动弹出浏览器页面。
 
+更多命令可以通过 `chatchat --help` 查看。
+
 ### 更新日志：
 #### 0.3.1.1 (2024-07-15)
 - 修复：

diff --git a/libs/chatchat-server/README_en.md b/libs/chatchat-server/README_en.md
@@ -17,11 +17,17 @@ Click [here](https://github.com/chatchat-space/Langchain-Chatchat）to Understan
 1. PYPI installation
 ```shell
 pip install langchain-chatchat
+
 # or if you use xinference to provide model API:
 # pip install langchain-chatchat[xinference]
+
+# if you update from an old version, we suggest to run init again to update yaml templates:
+# pip install -U langchain-chatchat
+# chatchat init
 ```
 Please refer to the [Installation Guide](https://github.com/chatchat-space/Langchain-Chatchat/tree/master?tab=readme-OVfile#%E5%BF%AB%E9%80%9F%E4%B8%8A%E6%89%8B) for details.
 >Attention: Chatchat should be placed in a separate virtual environment, such as conda, venv, virtualienv, etc
+
 >Known issue, cannot be installed together with xinference, which may cause some plugins to have bugs, such as file upload issues
 
 2. Source code installation
@@ -31,11 +37,13 @@ In addition to installing through Pypi, you can also choose to use [source code
 
 3. Docker
 ```shell
-docker pull chatimage/chatchat:0.3.0-2024-0624
+docker pull chatimage/chatchat:0.3.1.1-2024-0714
+
+docker pull ccr.ccs.tencentyun.com/chatchat/chatchat:0.3.1.1-2024-0714 # 国内镜像
 ```
 > [!important]
-> Strong recommendation: Use docker compose for deployment, refer to [README.docker](https://github.com/chatchat-space/Langchain-Chatchat/blob/master/docs/install/README.docker.md) for details 
-4. AudoDL
+> Strong recommendation: Use docker compose for deployment, refer to [README.docker](https://github.com/chatchat-space/Langchain-Chatchat/blob/master/docs/install/README_docker.md) for details 
+1. AudoDL
 🌐  [AutoDL Image](https://www.codewithgpu.com/i/chatchat-space/Langchain-Chatchat/Langchain-Chatchat）Medium ` 0.3.0`
 The code used in the version has been updated to version v0.3.0 of this project.
 
@@ -54,7 +62,7 @@ You can find the `*_ settings.yaml` files in CHATCHAT-ROOT or the current direct
 After ensuring that all configurations are correct (especially LLM and Embedding Model), execute the following commands to create the default knowledge base and start the service:
 ```shell
 chatchat kb -r
-chatchat -a
+chatchat start -a
 ```
 If there are no errors, the browser page will automatically pop up.
 ### Update log:

diff --git a/libs/chatchat-server/chatchat/cli.py b/libs/chatchat-server/chatchat/cli.py
@@ -3,6 +3,8 @@
 import shutil
 import typing as t
 
+import nltk
+
 from chatchat.startup import main as startup_main
 from chatchat.init_database import main as kb_main, create_tables, folder2db
 from chatchat.settings import Settings
@@ -50,9 +52,7 @@ def init(
     if(bs.PACKAGE_ROOT / "data/knowledge_base/samples" != Path(bs.KB_ROOT_PATH) / "samples"):
         shutil.copytree(bs.PACKAGE_ROOT / "data/knowledge_base/samples", Path(bs.KB_ROOT_PATH) / "samples", dirs_exist_ok=True)
     logger.info("复制 samples 知识库文件：成功。")
-    if (bs.PACKAGE_ROOT / "data/nltk_data" != bs.NLTK_DATA_PATH):
-        shutil.copytree(bs.PACKAGE_ROOT / "data/nltk_data", bs.NLTK_DATA_PATH, dirs_exist_ok=True)
-    logger.info("复制 nltl_data：成功。")
+    nltk.data.path.append(str(bs.PACKAGE_ROOT / "data/nltk_data"))
     create_tables()
     logger.info("初始化知识库数据库：成功。")
 
@@ -74,9 +74,9 @@ def init(
                   mode="recreate_vs",
                   vs_type=Settings.kb_settings.DEFAULT_VS_TYPE,
                   embed_model=get_default_embedding())
-        logger.success("所有初始化已完成，执行 chatchat start -a 启动服务。")
+        logger.success("<green>所有初始化已完成，执行 chatchat start -a 启动服务。</green>")
     else:
-        logger.warning("执行 chatchat kb -r 初始化知识库，然后 chatchat start -a 启动服务。")
+        logger.success("<green>执行 chatchat kb -r 初始化知识库，然后 chatchat start -a 启动服务。</green>")
 
 
 main.add_command(startup_main, "start")

diff --git a/libs/chatchat-server/chatchat/server/knowledge_base/migrate.py b/libs/chatchat-server/chatchat/server/knowledge_base/migrate.py
@@ -202,7 +202,6 @@ def files2vs(kb_name: str, kb_files: List[KnowledgeFile]) -> List:
             )
         )
         print("-" * 100 + "\n")
-        return result
 
 
 def prune_db_docs(kb_names: List[str]):

diff --git a/libs/chatchat-server/chatchat/server/utils.py b/libs/chatchat-server/chatchat/server/utils.py
@@ -140,9 +140,9 @@ def get_config_models(
                     logger.warning('auto_detect_model needs xinference-client installed. '
                                    'Please try "pip install xinference-client". ')
                 except requests.exceptions.ConnectionError:
-                    logger.error(f"cannot connect to xinference host: {xf_url}")
+                    logger.warning(f"cannot connect to xinference host: {xf_url}")
                 except Exception as e:
-                    logger.error(f"error when connect to xinference server({xf_url}): {e}")
+                    logger.warning(f"error when connect to xinference server({xf_url}): {e}")
 
         for m_type in model_types:
             models = m.get(m_type, [])

diff --git a/libs/chatchat-server/chatchat/utils.py b/libs/chatchat-server/chatchat/utils.py
@@ -6,9 +6,12 @@
 
 import loguru
 import loguru._logger
+from memoization import cached, CachingAlgorithmFlag
 from chatchat.settings import Settings
 
 
+# 默认每调用一次 build_logger 就会添加一次 hanlder，导致 chatchat.log 里重复输出
+@cached(max_size=100, algorithm=CachingAlgorithmFlag.LRU)
 def build_logger(log_file: str = "chatchat"):
     """
     build a logger with colorized output and a log file, for example:
-Original file line number
+Diff line change
@@ Expand Up @@
                 )
             )
             print("-" * 100 + "\n")
-            return result
     def prune_db_docs(kb_names: List[str]):
@@ Expand Down @@