try to fix deno issue + add similarity threshold for search

mediar-ai · Sep 27, 2024 · aef3426 · aef3426
1 parent c000fbd
commit aef3426
Show file tree

Hide file tree

Showing 13 changed files with 242 additions and 87 deletions.
diff --git a/screenpipe-app-tauri/bun.lockb b/screenpipe-app-tauri/bun.lockb
diff --git a/screenpipe-app-tauri/components/pipe-store.tsx b/screenpipe-app-tauri/components/pipe-store.tsx
@@ -35,7 +35,7 @@ import { PipeConfigForm } from "./pipe-config-form";
 import { useHealthCheck } from "@/lib/hooks/use-health-check";
 import posthog from "posthog-js";
 import { open } from "@tauri-apps/plugin-dialog";
-import { open as openUrl } from "@tauri-apps/plugin-shell";
+import { Command, open as openUrl } from "@tauri-apps/plugin-shell";
 import {
   Tooltip,
   TooltipContent,
@@ -101,7 +101,10 @@ const PipeDialog: React.FC = () => {
   const handleResetAllPipes = async () => {
     try {
       // reset pipes
-      await invoke("reset_all_pipes");
+      // await invoke("reset_all_pipes");
+      // instead use screenpipe pipe purge -y
+      const cmd = Command.sidecar("screenpipe", ["pipe", "purge", "-y"]);
+      await cmd.execute();
       await new Promise((resolve) => setTimeout(resolve, 1000));
       toast({
         title: "All pipes deleted",

diff --git a/screenpipe-app-tauri/components/search-chat.tsx b/screenpipe-app-tauri/components/search-chat.tsx
@@ -71,8 +71,9 @@ import { formatISO } from "date-fns";
 import { IconCode } from "@/components/ui/icons";
 import { CodeBlock } from "./ui/codeblock";
 import { SqlAutocompleteInput } from "./sql-autocomplete-input";
-import { encode } from "@/lib/utils";
+import { encode, removeDuplicateSelections } from "@/lib/utils";
 import { ExampleSearch, ExampleSearchCards } from "./example-search-cards";
+import { useDebounce } from "@/lib/hooks/use-debounce";
 
 export function SearchChat() {
   // Search state
@@ -118,6 +119,7 @@ export function SearchChat() {
   const [selectedResults, setSelectedResults] = useState<Set<number>>(
     new Set()
   );
+  const [similarityThreshold, setSimilarityThreshold] = useState(0.9);
   const [hoveredResult, setHoveredResult] = useState<number | null>(null);
 
   const [isCurlDialogOpen, setIsCurlDialogOpen] = useState(false);
@@ -131,6 +133,9 @@ export function SearchChat() {
 
   const [hasSearched, setHasSearched] = useState(false);
 
+  const [isFiltering, setIsFiltering] = useState(false);
+  const debouncedThreshold = useDebounce(similarityThreshold, 300);
+
   const handleExampleSelect = async (example: ExampleSearch) => {
     posthog.capture("example_search", { example: example.title });
 
@@ -191,6 +196,23 @@ export function SearchChat() {
     }
   }, [results]);
 
+  useEffect(() => {
+    handleFilterDuplicates();
+  }, [debouncedThreshold, results]);
+
+  const handleFilterDuplicates = async () => {
+    setIsFiltering(true);
+    // simulate a delay to show loading state
+    await new Promise((resolve) => setTimeout(resolve, 100));
+
+    const allIndices = new Set(results.map((_, index) => index));
+    setSelectedResults(
+      removeDuplicateSelections(results, allIndices, debouncedThreshold)
+    );
+    setSelectAll(false);
+    setIsFiltering(false);
+  };
+
   useEffect(() => {
     const handleScroll = () => {
       const currentScrollPosition = window.scrollY;
@@ -966,13 +988,47 @@ export function SearchChat() {
         </div>
       )}
       {results.length > 0 && (
-        <div className="flex items-center space-x-2 mb-4 my-8">
-          <Checkbox
-            id="select-all"
-            checked={selectAll}
-            onCheckedChange={handleSelectAll}
-          />
-          <Label htmlFor="select-all">select all results</Label>
+        <div className="flex flex-col space-y-4 mb-4 my-8">
+          <div className="flex justify-between items-center">
+            <div className="flex items-center space-x-2">
+              <Checkbox
+                id="select-all"
+                checked={selectAll}
+                onCheckedChange={handleSelectAll}
+              />
+              <Label htmlFor="select-all">select all results</Label>
+            </div>
+            <div className="flex items-center space-x-2">
+              <Label htmlFor="similarity-threshold" className="ml-4">
+                similarity threshold: {similarityThreshold.toFixed(2)}
+              </Label>
+              <TooltipProvider>
+                <Tooltip>
+                  <TooltipTrigger asChild>
+                    <HelpCircle className="h-4 w-4 text-gray-400 cursor-help" />
+                  </TooltipTrigger>
+                  <TooltipContent>
+                    <p>
+                      adjust this slider to unselect similar results. lower
+                      values mean stricter filtering.
+                    </p>
+                  </TooltipContent>
+                </Tooltip>
+              </TooltipProvider>
+              <div className="relative w-64">
+                <Slider
+                  id="similarity-threshold"
+                  min={0.5}
+                  max={1}
+                  step={0.01}
+                  value={[similarityThreshold]}
+                  onValueChange={(value) => setSimilarityThreshold(value[0])}
+                  className={isFiltering ? "opacity-50 cursor-not-allowed" : ""}
+                  disabled={isFiltering}
+                />
+              </div>
+            </div>
+          </div>
         </div>
       )}
       <div className="space-y-4">

diff --git a/screenpipe-app-tauri/lib/hooks/use-debounce.tsx b/screenpipe-app-tauri/lib/hooks/use-debounce.tsx
@@ -0,0 +1,17 @@
+import { useState, useEffect } from "react";
+
+export function useDebounce<T>(value: T, delay: number): T {
+  const [debouncedValue, setDebouncedValue] = useState<T>(value);
+
+  useEffect(() => {
+    const handler = setTimeout(() => {
+      setDebouncedValue(value);
+    }, delay);
+
+    return () => {
+      clearTimeout(handler);
+    };
+  }, [value, delay]);
+
+  return debouncedValue;
+}
diff --git a/screenpipe-app-tauri/lib/utils.ts b/screenpipe-app-tauri/lib/utils.ts
@@ -1,6 +1,8 @@
 import { platform } from "@tauri-apps/plugin-os";
 import { type ClassValue, clsx } from "clsx";
 import { twMerge } from "tailwind-merge";
+import { ContentItem } from "./screenpipe";
+import levenshtein from "js-levenshtein";
 
 export function cn(...inputs: ClassValue[]) {
   return twMerge(clsx(inputs));
@@ -58,3 +60,43 @@ export function getCliPath() {
       return "screenpipe";
   }
 }
+
+// Add this pure function outside of the SearchChat component
+export const removeDuplicateSelections = (
+  results: ContentItem[],
+  selectedResults: Set<number>,
+  similarityThreshold: number = 0.9
+): Set<number> => {
+  const newSelectedResults = new Set<number>();
+  const seenContents: string[] = [];
+
+  const getSimilarity = (str1: string, str2: string): number => {
+    const maxLength = Math.max(str1.length, str2.length);
+    const distance = levenshtein(str1, str2);
+    return 1 - distance / maxLength;
+  };
+
+  const isDuplicate = (content: string): boolean => {
+    return seenContents.some(
+      (seenContent) =>
+        getSimilarity(content, seenContent) >= similarityThreshold
+    );
+  };
+
+  Array.from(selectedResults).forEach((index) => {
+    const item = results[index];
+    if (!item || !item.type) return;
+
+    let content = "";
+    if (item.type === "OCR") content = item.content.text;
+    else if (item.type === "Audio") content = item.content.transcription;
+    else if (item.type === "FTS") content = item.content.matched_text;
+
+    if (!isDuplicate(content)) {
+      seenContents.push(content);
+      newSelectedResults.add(index);
+    }
+  });
+
+  return newSelectedResults;
+};
diff --git a/screenpipe-app-tauri/package.json b/screenpipe-app-tauri/package.json
@@ -42,13 +42,15 @@
     "@tauri-apps/plugin-shell": "2.0.0-beta.8",
     "@tauri-apps/plugin-store": "2.0.0-beta.7",
     "@tauri-apps/plugin-updater": "2.0.0-beta.7",
+    "@types/js-levenshtein": "^1.1.3",
     "ai": "^3.3",
     "ansi-to-html": "^0.7.2",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "cmdk": "1.0.0",
     "date-fns": "^3.6.0",
     "framer-motion": "^11.5.4",
+    "js-levenshtein": "^1.1.6",
     "localforage": "^1.10.0",
     "lodash": "^4.17.21",
     "lucide-react": "^0.414.0",

diff --git a/screenpipe-app-tauri/src-tauri/Cargo.toml b/screenpipe-app-tauri/src-tauri/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "screenpipe-app"
-version = "0.2.73"
+version = "0.2.74"
 description = ""
 authors = ["you"]
 license = ""

diff --git a/screenpipe-app-tauri/src-tauri/capabilities/main.json b/screenpipe-app-tauri/src-tauri/capabilities/main.json
@@ -118,6 +118,21 @@
         }
       ]
     },
+    {
+      "identifier": "shell:allow-execute",
+      "allow": [
+        {
+          "name": "screenpipe",
+          "cmd": "screenpipe",
+          "args": [
+            "pipe",
+            "purge",
+            "-y"
+          ],
+          "sidecar": true
+        }
+      ]
+    },
     "cli:default",
     "shell:default",
     "dialog:default",

diff --git a/screenpipe-app-tauri/src-tauri/gen/schemas/capabilities.json b/screenpipe-app-tauri/src-tauri/gen/schemas/capabilities.json
@@ -1 +1 @@
-{"migrated":{"identifier":"migrated","description":"permissions that were migrated from v1","local":true,"windows":["main"],"permissions":["core:event:allow-listen","core:event:default","process:allow-restart","process:allow-exit","notification:default","core:resources:default","core:menu:default","core:tray:default","shell:default","store:allow-get","store:allow-set","store:allow-save","store:allow-load","shell:allow-open","fs:allow-watch","fs:allow-open","fs:default","fs:allow-read-file","fs:allow-read-dir","fs:allow-read","fs:allow-write-file","fs:read-dirs","fs:allow-copy-file","fs:allow-exists","fs:allow-read-text-file","fs:allow-create","fs:allow-mkdir",{"identifier":"fs:scope","allow":[{"path":"$XDG_DATA_HOME","requireLiteralLeadingDot":false},{"path":"$XDG_DATA_HOME/**","requireLiteralLeadingDot":false},{"path":"$LOCALAPPDATA/*","requireLiteralLeadingDot":false},{"path":"$LOCALAPPDATA/**","requireLiteralLeadingDot":false},{"path":"$APPLOCALDATA/*","requireLiteralLeadingDot":false},{"path":"$APPLOCALDATA/**","requireLiteralLeadingDot":false},{"path":"$APPDATA/*","requireLiteralLeadingDot":false},{"path":"$APPDATA/**","requireLiteralLeadingDot":false},{"path":"$APPCONFIG/*","requireLiteralLeadingDot":false},{"path":"$RESOURCE/*","requireLiteralLeadingDot":false},{"path":"$RESOURCE/.screenpipe/*","requireLiteralLeadingDot":false},{"path":"$HOME/*","requireLiteralLeadingDot":false},{"path":"$HOME/.screenpipe/*","requireLiteralLeadingDot":false},{"path":"$APP/*","requireLiteralLeadingDot":false},{"path":"$HOME/.screenpipe/data/**","requireLiteralLeadingDot":false},{"path":"$HOME/.screenpipe/pipes/**","requireLiteralLeadingDot":false}]},{"identifier":"shell:allow-execute","allow":[{"args":["-c",{"validator":"\\S+"}],"cmd":"sh","name":"exec-sh","sidecar":false}]},"cli:default","shell:default","dialog:default","updater:default","os:default","os:allow-arch","os:allow-hostname","os:allow-os-type","core:path:default","process:default"]}}
+{"migrated":{"identifier":"migrated","description":"permissions that were migrated from v1","local":true,"windows":["main"],"permissions":["core:event:allow-listen","core:event:default","process:allow-restart","process:allow-exit","notification:default","core:resources:default","core:menu:default","core:tray:default","shell:default","store:allow-get","store:allow-set","store:allow-save","store:allow-load","shell:allow-open","fs:allow-watch","fs:allow-open","fs:default","fs:allow-read-file","fs:allow-read-dir","fs:allow-read","fs:allow-write-file","fs:read-dirs","fs:allow-copy-file","fs:allow-exists","fs:allow-read-text-file","fs:allow-create","fs:allow-mkdir",{"identifier":"fs:scope","allow":[{"path":"$XDG_DATA_HOME","requireLiteralLeadingDot":false},{"path":"$XDG_DATA_HOME/**","requireLiteralLeadingDot":false},{"path":"$LOCALAPPDATA/*","requireLiteralLeadingDot":false},{"path":"$LOCALAPPDATA/**","requireLiteralLeadingDot":false},{"path":"$APPLOCALDATA/*","requireLiteralLeadingDot":false},{"path":"$APPLOCALDATA/**","requireLiteralLeadingDot":false},{"path":"$APPDATA/*","requireLiteralLeadingDot":false},{"path":"$APPDATA/**","requireLiteralLeadingDot":false},{"path":"$APPCONFIG/*","requireLiteralLeadingDot":false},{"path":"$RESOURCE/*","requireLiteralLeadingDot":false},{"path":"$RESOURCE/.screenpipe/*","requireLiteralLeadingDot":false},{"path":"$HOME/*","requireLiteralLeadingDot":false},{"path":"$HOME/.screenpipe/*","requireLiteralLeadingDot":false},{"path":"$APP/*","requireLiteralLeadingDot":false},{"path":"$HOME/.screenpipe/data/**","requireLiteralLeadingDot":false},{"path":"$HOME/.screenpipe/pipes/**","requireLiteralLeadingDot":false}]},{"identifier":"shell:allow-execute","allow":[{"args":["-c",{"validator":"\\S+"}],"cmd":"sh","name":"exec-sh","sidecar":false}]},{"identifier":"shell:allow-execute","allow":[{"args":["pipe","purge","-y"],"cmd":"screenpipe","name":"screenpipe","sidecar":true}]},"cli:default","shell:default","dialog:default","updater:default","os:default","os:allow-arch","os:allow-hostname","os:allow-os-type","core:path:default","process:default"]}}
diff --git a/screenpipe-core/Cargo.toml b/screenpipe-core/Cargo.toml
@@ -22,8 +22,8 @@ hf-hub = { workspace = true, features = ["tokio"], optional = true }
 reqwest = { workspace = true }
 tokio = { workspace = true }
 
-deno_core = { version = "0.307.0", optional = true }
-deno_ast = { version = "0.38.2", features = ["transpiling"], optional = true }
+deno_core = { version = "0.311.0", optional = true }
+deno_ast = { version = "0.42.0", features = ["transpiling"], optional = true }
 # Security
 regex = { version = "1.10.6", features = ["std"], optional = true }
 lazy_static = { version = "1.4.0", optional = true }