Merge pull request #36 from SciPhi-AI/Nolan/RemoveDocumentsAddFolders

Remove documents endpoints, add folder support

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,17 +1,17 @@
 ---
 name: Bug report
 about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
+title: ""
+labels: ""
+assignees: ""
 ---
 
 **Describe the bug**
 A clear and concise description of what the bug is.
 
 **To Reproduce**
 Steps to reproduce the behavior:
+
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
@@ -24,15 +24,17 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+
+- OS: [e.g. iOS]
+- Browser [e.g. chrome, safari]
+- Version [e.g. 22]
 
 **Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
+
+- Device: [e.g. iPhone6]
+- OS: [e.g. iOS8.1]
+- Browser [e.g. stock browser, safari]
+- Version [e.g. 22]
 
 **Additional context**
 Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,10 +1,9 @@
 ---
 name: Feature request
 about: Suggest an idea for this project
-title: ''
-labels: ''
-assignees: ''
-
+title: ""
+labels: ""
+assignees: ""
 ---
 
 **Is your feature request related to a problem? Please describe.**

__tests__/r2rClientIntegration.test.ts

@@ -15,53 +15,29 @@ describe("r2rClient Integration Tests", () => {
     await expect(client.healthCheck()).resolves.not.toThrow();
   });
 
-  test("Ingest documents", async () => {
-    const documentsToIngest = [
-      {
-        type: "txt",
-        data: fs.readFileSync("examples/data/myshkin.txt", "utf8"),
-        metadata: { title: "myshkin.txt" },
-      },
-    ];
-    await expect(
-      client.ingestDocuments(documentsToIngest),
-    ).resolves.not.toThrow();
-  });
-
-  test("Update documents", async () => {
-    const documentsToUpdate = [
-      {
-        id: "4430ddbf-4323-5a14-9205-c092920e9321",
-        type: "txt",
-        data: fs.readFileSync("examples/data/raskolnikov.txt", "utf8"),
-        metadata: { title: "updated_myshkin.txt" },
-      },
-    ];
-    await expect(
-      client.updateDocuments(documentsToUpdate),
-    ).resolves.not.toThrow();
-  });
-
-  test("Ingest files", async () => {
+  test("Ingest file", async () => {
     const files = [
       { path: "examples/data/raskolnikov.txt", name: "raskolnikov.txt" },
-      { path: "examples/data/karamozov.txt", name: "karamozov.txt" },
     ];
+
     await expect(
       client.ingestFiles(files, {
         metadatas: [{ title: "raskolnikov.txt" }, { title: "karamozov.txt" }],
-        user_ids: [
-          "123e4567-e89b-12d3-a456-426614174000",
-          "123e4567-e89b-12d3-a456-426614174000",
-        ],
+        user_ids: ["123e4567-e89b-12d3-a456-426614174000"],
         skip_document_info: false,
       }),
     ).resolves.not.toThrow();
   });
 
+  test("Ingest files in folder", async () => {
+    const files = ["examples/data/folder"];
+
+    await expect(client.ingestFiles(files)).resolves.not.toThrow();
+  });
+
   test("Update files", async () => {
     const updated_file = [
-      { path: "examples/data/myshkin.txt", name: "myshkin.txt" },
+      { path: "examples/data/folder/myshkin.txt", name: "myshkin.txt" },
     ];
     await expect(
       client.updateFiles(updated_file, {
@@ -131,7 +107,7 @@ describe("r2rClient Integration Tests", () => {
       ["document_id", "document_id"],
       [
         "48e29904-3010-54fe-abe5-a4f3fba59110",
-        "4430ddbf-4323-5a14-9205-c092920e9321",
+        "102e815f-3998-5373-8d7d-a07795266ed6",
       ],
     );
   });

package.json

@@ -1,6 +1,6 @@
 {
   "name": "r2r-js",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "",
   "main": "dist/index.js",
   "browser": "dist/index.browser.js",

src/models.tsx

@@ -49,17 +49,6 @@ export interface R2RUpdatePromptRequest {
   input_types?: Record<string, string>;
 }
 
-export interface R2RIngestDocumentsRequest {
-  documents: Document[];
-  versions?: string[];
-}
-
-export interface R2RUpdateDocumentsRequest {
-  documents: Document[];
-  versions?: string[] | null;
-  metadatas?: Record<string, any>[] | null;
-}
-
 export interface R2RIngestFilesRequest {
   metadatas?: Record<string, any>[];
   document_ids?: string[];

src/r2rClient.ts

@@ -1,5 +1,6 @@
 import axios, { AxiosInstance } from "axios";
 import FormData from "form-data";
+import * as path from "path";
 
 let fs: any;
 if (typeof window === "undefined") {
@@ -10,9 +11,7 @@ import { feature, initializeTelemetry } from "./feature";
 import {
   Document,
   R2RUpdatePromptRequest,
-  R2RIngestDocumentsRequest,
   R2RIngestFilesRequest,
-  R2RUpdateDocumentsRequest,
   R2RSearchRequest,
   R2RRAGRequest,
   R2RDeleteRequest,
@@ -85,33 +84,9 @@ export class r2rClient {
     return response.data;
   }
 
-  @feature("ingestDocuments")
-  async ingestDocuments(
-    documents: Document[],
-    versions?: string[],
-  ): Promise<any> {
-    const processedDocuments = documents.map((doc) => ({
-      type: doc.type,
-      data: Buffer.from(doc.data).toString("base64"),
-      metadata: doc.metadata,
-      ...(doc.id && { id: doc.id }),
-    }));
-
-    const request: R2RIngestDocumentsRequest = {
-      documents: processedDocuments,
-      ...(versions && { versions }),
-    };
-
-    const response = await this.axiosInstance.post(
-      "/ingest_documents",
-      request,
-    );
-    return response.data;
-  }
-
   @feature("ingestFiles")
   async ingestFiles(
-    files: (File | { path: string; name: string })[],
+    files: (string | File | { path: string; name: string })[],
     options: {
       metadatas?: Record<string, any>[];
       document_ids?: string[];
@@ -122,19 +97,48 @@ export class r2rClient {
   ): Promise<any> {
     const formData = new FormData();
 
-    files.forEach((file, index) => {
-      if ("path" in file) {
+    const processPath = async (
+      path: string | File | { path: string; name: string },
+      index: number,
+    ) => {
+      if (typeof path === "string") {
+        if (typeof window === "undefined") {
+          const stat = await fs.promises.stat(path);
+          if (stat.isDirectory()) {
+            const files = await fs.promises.readdir(path, {
+              withFileTypes: true,
+            });
+            for (const file of files) {
+              await processPath(`${path}/${file.name}`, index);
+            }
+          } else {
+            formData.append(
+              "files",
+              fs.createReadStream(path),
+              path.split("/").pop(),
+            );
+          }
+        } else {
+          console.warn(
+            "File or folder path provided in browser environment. This is not supported.",
+          );
+        }
+      } else if (path instanceof File) {
+        formData.append("files", path);
+      } else if ("path" in path) {
         if (typeof window === "undefined") {
-          formData.append("files", fs.createReadStream(file.path), file.name);
+          formData.append("files", fs.createReadStream(path.path), path.name);
         } else {
           console.warn(
             "File path provided in browser environment. This is not supported.",
           );
         }
-      } else {
-        formData.append("files", file);
       }
-    });
+    };
+
+    for (let i = 0; i < files.length; i++) {
+      await processPath(files[i], i);
+    }
 
     const request: R2RIngestFilesRequest = {
       metadatas: options.metadatas,
@@ -164,39 +168,6 @@ export class r2rClient {
     return response.data;
   }
 
-  @feature("updateDocuments")
-  async updateDocuments(
-    documents: Document[],
-    versions?: string[] | null,
-    metadatas?: Record<string, any>[] | null,
-  ): Promise<any> {
-    const processedDocuments = documents.map((doc) => ({
-      id: doc.id,
-      type: doc.type,
-      data: Buffer.from(doc.data).toString("base64"),
-      metadata: doc.metadata,
-    }));
-
-    const request: R2RUpdateDocumentsRequest = {
-      documents: processedDocuments,
-    };
-
-    // Only include versions and metadatas if they're explicitly provided
-    if (versions !== undefined && versions !== null) {
-      request.versions = versions;
-    }
-
-    if (metadatas !== undefined && metadatas !== null) {
-      request.metadatas = metadatas;
-    }
-
-    const response = await this.axiosInstance.post(
-      "/update_documents",
-      request,
-    );
-    return response.data;
-  }
-
   @feature("updateFiles")
   async updateFiles(
     files: (File | { path: string; name: string })[],