feat(project): support scanning interruption (#6)

.github/workflows/build.yml

@@ -5,15 +5,48 @@ on:
       - "main"
 
 jobs:
+  build-browser:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 8
+          run_install: false
+
+      - name: NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: pnpm
+          cache-dependency-path: ./browser/pnpm-lock.yaml
+
+      - name: Install dependencies
+        working-directory: ./browser
+        run: pnpm i
+
+      - name: Check TypeScript
+        working-directory: ./browser
+        run: pnpm ts-check
+
+      - name: Build
+        working-directory: ./browser
+        run: pnpm build
+
   test:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.12.7"
+          cache: "pip"
 
       - name: Install dependencies
         run: |

browser/src/store/scanning_store.ts

@@ -2,7 +2,13 @@ import { val, derive, combine, Val, ReadonlyVal } from "value-enhancer";
 import { fetchJson, fetchJsonEvents, EventFetcher } from "../utils";
 
 type Event = {
-  readonly kind: "scanning" | "completed" | "interrupted" | "heartbeat";
+  readonly kind: (
+    "scanning" |
+    "completed" |
+    "interrupting" |
+    "interrupted" |
+    "heartbeat"
+  );
 } | {
   readonly kind: "scanCompleted";
   readonly count: number;
@@ -31,6 +37,8 @@ export type ScanningStore$ = {
   readonly handlingFile: ReadonlyVal<HandingFile | null>;
   readonly completedFiles: ReadonlyVal<readonly string[]>;
   readonly error: ReadonlyVal<string | null>;
+  readonly isInterrupting: ReadonlyVal<boolean>;
+  readonly isInterrupted: ReadonlyVal<boolean>;
   readonly isScanning: ReadonlyVal<boolean>;
 };
 
@@ -39,7 +47,6 @@ export enum ScanningPhase {
   Scanning,
   HandingFiles,
   Completed,
-  Error,
 }
 
 export type HandingFile = {
@@ -63,6 +70,8 @@ export class ScanningStore {
   readonly #handlingFile$: Val<HandingFile | null> = val<HandingFile | null>(null);
   readonly #completedFiles$: Val<readonly string[]> = val<readonly string[]>([]);
   readonly #error$: Val<string | null> = val<string | null>(null);
+  readonly #isInterrupting$: Val<boolean> = val(false);
+  readonly #isInterrupted$: Val<boolean> = val(false);
 
   public constructor() {
     this.#fetcher = fetchJsonEvents<Event>("/api/scanning");
@@ -72,25 +81,37 @@ export class ScanningStore {
       handlingFile: derive(this.#handlingFile$),
       completedFiles: derive(this.#completedFiles$),
       error: derive(this.#error$),
-      isScanning: derive(this.#phase$, phase => {
-        switch (phase) {
-          case ScanningPhase.Ready:
-          case ScanningPhase.Completed:
-          case ScanningPhase.Error: {
+      isInterrupting: derive(this.#isInterrupting$),
+      isInterrupted: derive(this.#isInterrupted$),
+      isScanning: combine(
+        [this.#phase$, this.#error$, this.#isInterrupted$],
+        ([phase, error, isInterrupted]) => {
+          switch (phase) {
+            case ScanningPhase.Ready:
+            case ScanningPhase.Completed: {
+              return false;
+            }
+          }
+          if (error) {
             return false;
           }
-          default: {
-            return true;
+          if (isInterrupted) {
+            return false;
           }
-        }
-      }),
+          return true;
+        },
+      ),
     };
   }
 
   public async scan(): Promise<void> {
     await fetchJson("/api/scanning", { method: "POST" });
   }
 
+  public async interrupt(): Promise<void> {
+    await fetchJson("/api/scanning", { method: "DELETE" });
+  }
+
   public close(): void {
     this.#fetcher.close();
   }
@@ -108,6 +129,7 @@ export class ScanningStore {
           this.#handlingFile$.set(null);
           this.#completedFiles$.set([]);
           this.#error$.set(null);
+          this.#isInterrupted$.set(false);
           break;
         }
         case "scanCompleted": {
@@ -152,10 +174,18 @@ export class ScanningStore {
           break;
         }
         case "failure": {
-          this.#phase$.set(ScanningPhase.Error);
           this.#error$.set(event.error);
           break;
         }
+        case "interrupting": {
+          this.#isInterrupting$.set(true);
+          break;
+        }
+        case "interrupted": {
+          this.#isInterrupting$.set(false);
+          this.#isInterrupted$.set(true);
+          break;
+        }
       }
     }
   }

browser/src/views/ScannerPage.module.less

@@ -5,6 +5,7 @@
 
 .scan-button {
   width: 128px;
+  margin-right: 24px;
 }
 
 .steps-bar {

browser/src/views/ScannerPage.tsx

@@ -2,7 +2,7 @@ import React from "react";
 import styles from "./ScannerPage.module.less";
 
 import { Skeleton, Result, Steps, List, Button, Divider, Progress, Typography } from "antd";
-import { ScanOutlined, ProfileTwoTone, SyncOutlined, FilePdfTwoTone } from "@ant-design/icons";
+import { ScanOutlined, ProfileTwoTone, SyncOutlined, FilePdfTwoTone, PauseOutlined } from "@ant-design/icons";
 import { val } from "value-enhancer";
 import { useVal } from "use-value-enhancer";
 import { ScannerStore, ScanningStore, ScanningPhase } from "../store";
@@ -49,10 +49,15 @@ type ScannerProps = {
 const Scanner: React.FC<ScannerProps> = ({ store }) => {
   const scanningStore = store.scanningStore;
   const isScanning = useVal(scanningStore.$.isScanning);
+  const isInterrupting = useVal(scanningStore.$.isInterrupting);
   const onClickScan = React.useCallback(
     () => scanningStore.scan(),
     [scanningStore],
   );
+  const onClickInterrupt = React.useCallback(
+    () => scanningStore.interrupt(),
+    [scanningStore],
+  );
   return <>
     <Typography>
       <Title>扫描</Title>
@@ -71,6 +76,18 @@ const Scanner: React.FC<ScannerProps> = ({ store }) => {
       onClick={onClickScan} >
       扫  描
     </Button>
+    {isScanning && (
+      <Button
+        shape="round"
+        size="large"
+        className={styles["scan-button"]}
+        disabled={isInterrupting}
+        loading={isInterrupting}
+        icon={<PauseOutlined />}
+        onClick={onClickInterrupt} >
+        中 断
+      </Button>
+    )}
   </>;
 }
 
@@ -91,6 +108,7 @@ const ScanningPanel: React.FC<ScanningPanelProps> = ({ store }) => {
   const completedFiles = useVal(store.$.completedFiles);
   const handlingFile = useVal(store.$.handlingFile);
   const error = useVal(store.$.error);
+  const isInterrupted = useVal(store.$.isInterrupted);
 
   let currentIndex: number;
   let status: "wait" | "process" | "finish" | "error" = "process";
@@ -112,11 +130,6 @@ const ScanningPanel: React.FC<ScanningPanelProps> = ({ store }) => {
       status = "finish";
       break;
     }
-    case ScanningPhase.Error: {
-      currentIndex = 2;
-      status = "error";
-      break;
-    }
   }
   if (phase === ScanningPhase.Scanning) {
     records.push({
@@ -157,7 +170,7 @@ const ScanningPanel: React.FC<ScanningPanelProps> = ({ store }) => {
       items={[
         { title: "扫描" },
         { title: "处理文件" },
-        { title: phase === ScanningPhase.Error ? "错误" : "完成" },
+        { title: "完成" },
       ]}
     />
     <List
@@ -178,9 +191,11 @@ const ScanningPanel: React.FC<ScanningPanelProps> = ({ store }) => {
     />
     <ProgressBar
       name="解析"
+      error={!!error || isInterrupted}
       pdfPage={handlingFile?.handlePdfPage} />
     <ProgressBar
       name="索引"
+      error={!!error || isInterrupted}
       pdfPage={handlingFile?.indexPdfPage} />
     {error && (
       <Result
@@ -189,24 +204,32 @@ const ScanningPanel: React.FC<ScanningPanelProps> = ({ store }) => {
         subTitle={error}
       />
     )}
+    {isInterrupted && (
+      <Result
+        status="info"
+        title="扫描中断"
+        subTitle="你可以继续使用知识库，但一些信息可能无法读取，另一些已失效的信息可能被错误读取。如果你希望知识库能正常运行，请继续扫描并等待完成。"
+      />
+    )}
   </>;
 };
 
 type ProgressBarProps = {
   readonly name: string;
+  readonly error: boolean;
   readonly pdfPage?: {
     readonly index: number;
     readonly total: number;
   };
 };
 
-const ProgressBar: React.FC<ProgressBarProps> = ({ name, pdfPage }) => {
+const ProgressBar: React.FC<ProgressBarProps> = ({ name, error, pdfPage }) => {
   if (!pdfPage) {
     return null;
   }
   const { index, total } = pdfPage;
   const percent = Math.floor(Math.min(index / total, 1.0) * 100);
-  const status = percent === 100 ? "success" : "active";
+  const status = error ? "exception" : (percent === 100 ? "success" : "active");
   return (
     <div className={styles["progress"]}>
       <label className={styles["progress-label"]}>

browser/src/views/Sources.tsx

@@ -149,13 +149,15 @@ const SourceAddition: React.FC<SourceAdditionProps> = ({ store }) => {
             该名字已存在！
           </>}>
           <Input
+            placeholder="名字"
             className={styles["source-input-name"]}
             value={addedSourceName}
             disabled={isSubmittingAddition}
             status={isAddedNameDuplicated ? "error" : ""}
             onChange={onChangeNameInput} />
         </Popover>
         <Input
+          placeholder="知识库文件夹路径"
           ref={pathInputRef}
           value={addedSourcePath}
           disabled={isSubmittingAddition}

index_package/index/vector_db.py

@@ -9,8 +9,8 @@
 from chromadb.api.types import ID, EmbeddingFunction, IncludeEnum, Documents, Embedding, Embeddings, Document, Metadata
 from chromadb.utils import distance_functions
 
+from ..segmentation import Segment
 from .types import IndexNode, IndexSegment, IndexNodeMatching
-from ..segmentation.segmentation import Segment
 
 _DistanceFunction = Callable[[ArrayLike, ArrayLike], float]
 DistanceSpace = Literal["l2", "ip", "cosine"]