diff --git a/browser/src/store/index.ts b/browser/src/store/index.ts index 8acec02..e8f206c 100644 --- a/browser/src/store/index.ts +++ b/browser/src/store/index.ts @@ -1 +1,2 @@ -export * from "./scanner_store"; \ No newline at end of file +export * from "./scanner_store"; +export * from "./scanning_store"; \ No newline at end of file diff --git a/browser/src/store/scanning_store.ts b/browser/src/store/scanning_store.ts index 6d35391..7bc92f5 100644 --- a/browser/src/store/scanning_store.ts +++ b/browser/src/store/scanning_store.ts @@ -1,3 +1,4 @@ +import { val, derive, combine, Val, ReadonlyVal } from "value-enhancer"; import { fetchJson, fetchJsonEvents, EventFetcher } from "../utils"; type Event = { @@ -5,15 +6,12 @@ type Event = { } | { readonly kind: "scanCompleted"; readonly count: number; -} | { - readonly kind: "completeHandingFile"; - readonly path: string; -} | { - readonly kind: "failure"; - readonly error: string; } | { readonly kind: "startHandingFile"; readonly path: string; +}| { + readonly kind: "completeHandingFile"; + readonly path: string; } | { readonly kind: "completeHandingPdfPage"; readonly index: number; @@ -22,13 +20,71 @@ type Event = { readonly kind: "completeIndexPdfPage"; readonly index: number; readonly total: number; +} | { + readonly kind: "failure"; + readonly error: string; +}; + +export type ScanningStore$ = { + readonly phase: ReadonlyVal; + readonly scanCount: ReadonlyVal; + readonly handlingFile: ReadonlyVal; + readonly completedFiles: ReadonlyVal; + readonly error: ReadonlyVal; + readonly isScanning: ReadonlyVal; +}; + +export enum ScanningPhase { + Ready, + Scanning, + HandingFiles, + Completed, + Error, +} + +export type HandingFile = { + readonly path: string; + readonly handlePdfPage?: { + readonly index: number; + readonly total: number; + }; + readonly indexPdfPage?: { + readonly index: number; + readonly total: number; + }; }; export class ScanningStore { + public readonly $: ScanningStore$; + readonly #fetcher: EventFetcher; + readonly #phase$: Val = val(ScanningPhase.Ready); + readonly #scanCount$: Val = val(0); + readonly #handlingFile$: Val = val(null); + readonly #completedFiles$: Val = val([]); + readonly #error$: Val = val(null); public constructor() { this.#fetcher = fetchJsonEvents("/api/scanning"); + this.$ = { + phase: derive(this.#phase$), + scanCount: derive(this.#scanCount$), + handlingFile: derive(this.#handlingFile$), + completedFiles: derive(this.#completedFiles$), + error: derive(this.#error$), + isScanning: derive(this.#phase$, phase => { + switch (phase) { + case ScanningPhase.Ready: + case ScanningPhase.Completed: + case ScanningPhase.Error: { + return false; + } + default: { + return true; + } + } + }), + }; } public async scan(): Promise { @@ -45,7 +101,62 @@ export class ScanningStore { if (!event) { break; } - console.log("event", event); + switch (event.kind) { + case "scanning": { + this.#phase$.set(ScanningPhase.Scanning); + this.#scanCount$.set(0); + this.#handlingFile$.set(null); + this.#completedFiles$.set([]); + this.#error$.set(null); + break; + } + case "scanCompleted": { + this.#phase$.set(ScanningPhase.HandingFiles); + this.#scanCount$.set(event.count); + break; + } + case "startHandingFile": { + this.#handlingFile$.set({ path: event.path }); + break; + } + case "completeHandingFile": { + this.#handlingFile$.set(null); + this.#completedFiles$.set([ + ...this.#completedFiles$.value, + event.path, + ]); + break; + } + case "completeHandingPdfPage": { + this.#handlingFile$.set({ + ...this.#handlingFile$.value!, + handlePdfPage: { + index: event.index, + total: event.total, + }, + }); + break; + } + case "completeIndexPdfPage": { + this.#handlingFile$.set({ + ...this.#handlingFile$.value!, + indexPdfPage: { + index: event.index, + total: event.total, + }, + }); + break; + } + case "completed": { + this.#phase$.set(ScanningPhase.Completed); + break; + } + case "failure": { + this.#phase$.set(ScanningPhase.Error); + this.#error$.set(event.error); + break; + } + } } } } \ No newline at end of file diff --git a/browser/src/views/ScannerPage.module.less b/browser/src/views/ScannerPage.module.less index a6c128f..840c34b 100644 --- a/browser/src/views/ScannerPage.module.less +++ b/browser/src/views/ScannerPage.module.less @@ -1,3 +1,32 @@ +.root { + margin-bottom: 68px; + overflow-y: scroll; +} + .scan-button { width: 128px; +} + +.steps-bar { + margin-top: 75px; + margin-bottom: 28px; +} + +.progress { + display: flex; + flex-direction: row; + align-items: center; +} + +.progress-bar { + flex-basis: 340px; + flex-shrink: 1.0; +} + +.progress-label { + color: #8C8C8C; + flex-basis: 48px; + font-size: 12px; + margin-top: 18px; + margin-bottom: 18px; } \ No newline at end of file diff --git a/browser/src/views/ScannerPage.tsx b/browser/src/views/ScannerPage.tsx index 874db7b..76c7752 100644 --- a/browser/src/views/ScannerPage.tsx +++ b/browser/src/views/ScannerPage.tsx @@ -1,11 +1,11 @@ import React from "react"; import styles from "./ScannerPage.module.less"; -import { Skeleton, Button, Divider, Typography } from "antd"; -import { ScanOutlined } from "@ant-design/icons"; +import { Skeleton, Result, Steps, List, Button, Divider, Progress, Typography } from "antd"; +import { ScanOutlined, ProfileTwoTone, SyncOutlined, FilePdfTwoTone } from "@ant-design/icons"; import { val } from "value-enhancer"; import { useVal } from "use-value-enhancer"; -import { ScannerStore } from "../store"; +import { ScannerStore, ScanningStore, ScanningPhase } from "../store"; import { Sources } from "./Sources"; const { Title, Paragraph } = Typography; @@ -33,10 +33,11 @@ export const ScannerPage: React.FC = () => { return ; } return ( -
+
+
); }; @@ -47,6 +48,7 @@ type ScannerProps = { const Scanner: React.FC = ({ store }) => { const scanningStore = store.scanningStore; + const isScanning = useVal(scanningStore.$.isScanning); const onClickScan = React.useCallback( () => scanningStore.scan(), [scanningStore], @@ -63,9 +65,158 @@ const Scanner: React.FC = ({ store }) => { shape="round" size="large" className={styles["scan-button"]} + disabled={isScanning} + loading={isScanning} icon={} onClick={onClickScan} > 扫 描 ; -} \ No newline at end of file +} + +type ScanningPanelProps = { + readonly store: ScanningStore; +}; + +const ScanningPanel: React.FC = ({ store }) => { + type Record = { + readonly icon: React.ReactNode; + readonly title: string; + readonly content: string; + readonly loading: boolean; + }; + const records: Record[] = []; + const phase = useVal(store.$.phase); + const scanCount = useVal(store.$.scanCount); + const completedFiles = useVal(store.$.completedFiles); + const handlingFile = useVal(store.$.handlingFile); + const error = useVal(store.$.error); + + let currentIndex: number; + let status: "wait" | "process" | "finish" | "error" = "process"; + + switch (phase) { + case ScanningPhase.Ready: { + return null; + } + case ScanningPhase.Scanning: { + currentIndex = 0; + break; + } + case ScanningPhase.HandingFiles: { + currentIndex = 1; + break; + } + case ScanningPhase.Completed: { + currentIndex = 2; + status = "finish"; + break; + } + case ScanningPhase.Error: { + currentIndex = 2; + status = "error"; + break; + } + } + if (phase === ScanningPhase.Scanning) { + records.push({ + icon: , + title: "扫描文件", + content: `正在扫描文件的更新……`, + loading: true, + }); + } else { + records.push({ + icon: , + title: "扫描文件", + content: `扫描完成,发现 ${scanCount} 个文件有更新`, + loading: false, + }); + } + for (const file of completedFiles) { + records.push({ + icon: , + title: "录入 PDF 文件", + content: file, + loading: false, + }); + } + if (handlingFile) { + records.push({ + icon: , + title: "录入 PDF 文件", + content: handlingFile.path, + loading: true, + }); + } + return <> + + ( + + + {item.loading && ( + + )} + + )} + /> + + + {error && ( + + )} + ; +}; + +type ProgressBarProps = { + readonly name: string; + readonly pdfPage?: { + readonly index: number; + readonly total: number; + }; +}; + +const ProgressBar: React.FC = ({ name, pdfPage }) => { + if (!pdfPage) { + return null; + } + const { index, total } = pdfPage; + const percent = Math.floor(Math.min(index / total, 1.0) * 100); + const status = percent === 100 ? "success" : "active"; + return ( +
+ + +
+ ); +}; \ No newline at end of file diff --git a/browser/src/views/Sources.tsx b/browser/src/views/Sources.tsx index 556b683..5a47177 100644 --- a/browser/src/views/Sources.tsx +++ b/browser/src/views/Sources.tsx @@ -132,6 +132,10 @@ const SourceAddition: React.FC = ({ store }) => { }, [canAdd, store], ); + const onClickAdd = React.useCallback( + () => store.addSource(), + [store], + ); return (
@@ -163,7 +167,8 @@ const SourceAddition: React.FC = ({ store }) => { className={styles["source-button"]} type="primary" icon={} - disabled={!canAdd || isSubmittingAddition} > + disabled={!canAdd || isSubmittingAddition} + onClick={onClickAdd} > 添加
diff --git a/index_package/index/index.py b/index_package/index/index.py index ed25d87..bcfa8ef 100644 --- a/index_package/index/index.py +++ b/index_package/index/index.py @@ -154,6 +154,9 @@ def handle_event(self, event: Event, progress: Optional[Progress] = None): if path is None: return + if progress is not None: + progress.start_handle_file(path) + cursor = self._conn.cursor() try: cursor.execute("BEGIN TRANSACTION") @@ -178,6 +181,9 @@ def handle_event(self, event: Event, progress: Optional[Progress] = None): self._conn.commit() cursor.close() + if progress is not None: + progress.complete_handle_file(path) + except Exception as e: self._conn.rollback() raise e @@ -256,7 +262,12 @@ def _handle_found_pdf_hash(self, cursor: sqlite3.Cursor, hash: str, path: str, p assert_continue() if progress is not None: - progress.on_complete_index_pdf_page(page.index, len(pdf.pages)) + pages_count = len(pdf.pages) + progress.complete_index_pdf_page(page.index, pages_count) + + if progress is not None: + pages_count = len(pdf.pages) + progress.complete_index_pdf_page(pages_count, pages_count) except InterruptException as e: index_context.rollback() diff --git a/index_package/index/vector_db.py b/index_package/index/vector_db.py index f8f5950..42172da 100644 --- a/index_package/index/vector_db.py +++ b/index_package/index/vector_db.py @@ -155,8 +155,7 @@ def remove(self, node_id: str): group_size: int = 45 for offset in range(0, segments_len, group_size): - remain_count = segments_len - offset * group_size - ids_len = min(group_size, remain_count) + ids_len = min(group_size, segments_len - offset) ids = [f"{node_id}/{offset + i}" for i in range(ids_len)] self._db.delete(ids=ids) diff --git a/index_package/parser/pdf.py b/index_package/parser/pdf.py index ae2ce25..8076d94 100644 --- a/index_package/parser/pdf.py +++ b/index_package/parser/pdf.py @@ -233,6 +233,10 @@ def _create_and_split_pdf(self, cursor: sqlite3.Cursor, hash: str, file_path: st pages_count = len(added_page_hashes) progress.complete_handle_pdf_page(i, pages_count) + if progress is not None: + pages_count = len(added_page_hashes) + progress.complete_handle_pdf_page(pages_count, pages_count) + self._conn.commit() return pdf_id, metadata diff --git a/index_package/progress.py b/index_package/progress.py index 0c6fbd7..26f96e2 100644 --- a/index_package/progress.py +++ b/index_package/progress.py @@ -14,5 +14,5 @@ def __init__(self, listeners: ProgressListeners = ProgressListeners()) -> None: self.after_scan: Callable[[int], None] = listeners.after_scan or (lambda _: None) self.start_handle_file: Callable[[str], None] = listeners.on_start_handle_file or (lambda _: None) self.complete_handle_pdf_page: Callable[[int, int], None] = listeners.on_complete_handle_pdf_page or (lambda _1, _2: None) - self.on_complete_index_pdf_page: Callable[[int, int], None] = listeners.on_complete_index_pdf_page or (lambda _1, _2: None) + self.complete_index_pdf_page: Callable[[int, int], None] = listeners.on_complete_index_pdf_page or (lambda _1, _2: None) self.complete_handle_file: Callable[[str], None] = listeners.on_complete_handle_file or (lambda _: None) \ No newline at end of file diff --git a/index_package/service/scan_job.py b/index_package/service/scan_job.py index f6ca6cc..58f8f04 100644 --- a/index_package/service/scan_job.py +++ b/index_package/service/scan_job.py @@ -2,6 +2,7 @@ import threading from typing import cast, Optional, Callable + from .service_in_thread import ServiceInThread from ..scanner import Scope, EventParser, Scanner from ..progress import Progress @@ -90,10 +91,4 @@ def _handle_event(self, event_id: int, index: int): else: display_path = f"[removed]:{display_path}" - if self._progress is not None: - self._progress.start_handle_file(display_path) - service.handle_event(event, self._progress) - - if self._progress is not None: - self._progress.complete_handle_file(display_path) diff --git a/server/progress_events.py b/server/progress_events.py index 3ede14a..8d683f1 100644 --- a/server/progress_events.py +++ b/server/progress_events.py @@ -40,13 +40,14 @@ def listeners(self) -> ProgressListeners: on_complete_handle_file=self._on_complete_handle_file, ) - def reset(self): + def notify_scanning(self): with self._status_lock: + if self._phase != ProgressPhase.READY: + self._scanned_count = 0 + self._handing_file = None + self._error = None + self._completed_files.clear() self._phase = ProgressPhase.SCANNING - self._scanned_count = 0 - self._handing_file = None - self._error = None - self._completed_files.clear() self._emit_event({ "kind": "scanning", @@ -155,7 +156,6 @@ def _on_complete_index_pdf_page(self, page_index: int, total_pages: int): def complete(self): with self._status_lock: self._phase = ProgressPhase.COMPLETED - self._completed_files.clear() self._handing_file = None self._emit_event({ diff --git a/server/service.py b/server/service.py index 77a20b5..60892fa 100644 --- a/server/service.py +++ b/server/service.py @@ -47,7 +47,7 @@ def start_scanning(self): raise e def _scan(self): - self._progress_events.reset() + self._progress_events.notify_scanning() service = Service( workspace_path=self._workspace_path, embedding_model_id=self._embedding_model,