Support for subcategory backfill (redo)

README.md

@@ -63,7 +63,7 @@ When you install this extension, you'll be able to configure the following param
 
 | Parameter                   | Description                                                                                                                                                                                                                                                                                    |
 |-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Firestore Collection Path   | The Firestore collection that needs to be indexed into Typesense.                                                                                                                                                                                                                              |
+| Firestore Collection Path   | The Firestore collection that needs to be indexed into Typesense. Supports [subcollection wildcards](https://firebase.google.com/docs/functions/firestore-events?gen=2nd#wildcards-parameters) the following way: `users/{userId}/orders` where `{userId}` is a wildcard for all documents in the parent collection.                                                                                                                                                                                                                              |
 | Firestore Collection Fields | A comma separated list of fields that need to be indexed from each Firestore document. Leave blank to index all fields.                                                                                                                                                                        |
 | Flatten Nested Documents    | Should nested documents in Firestore be flattened before they are indexed in Typesense? Set to "Yes" for Typesense Server versions v0.23.1 and below, since indexing Nested objects is natively supported only in Typesense Server v0.24 and above.                                            |
 | Typesense Hosts             | A comma-separated list of Typesense Hosts (only domain without https or port number). For single node clusters, a single hostname is sufficient. For multi-node Highly Available or (Search Delivery Network) SDN Clusters, please be sure to mention all hostnames in a comma-separated list. | 

functions/src/backfill.js

@@ -41,7 +41,16 @@ module.exports = functions.firestore.document(config.typesenseBackfillTriggerDoc
 
   const typesense = createTypesenseClient();
 
-  const querySnapshot = await admin.firestore().collection(config.firestoreCollectionPath);
+  const isSubcollection = config.firestoreCollectionPath.includes("/");
+  let query;
+  if (isSubcollection) {
+    const collectionGroup = config.firestoreCollectionPath.split("/").pop();
+    query = admin.firestore().collectionGroup(collectionGroup);
+  } else {
+    query = admin.firestore().collection(config.firestoreCollectionPath);
+  }
+
+  const querySnapshot = await query;
 
   let lastDoc = null;
 

package.json

@@ -7,6 +7,7 @@
     "test": "npm run test-part-1 && npm run test-part-2",
     "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=\"WithoutFlattening\"'",
     "test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'",
+    "test-part-3": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"SubcollectionWithoutFlattening\"'",
     "typesenseServer": "docker compose up",
     "lint:fix": "eslint . --fix",
     "lint": "eslint ."

test/indexOnWriteSubcollectionWithoutFlattening.spec.js

@@ -0,0 +1,120 @@
+const firebase = require("firebase-admin");
+const config = require("../functions/src/config.js");
+const typesense = require("../functions/src/createTypesenseClient.js")();
+
+const app = firebase.initializeApp({
+  // Use a special URL to talk to the Realtime Database emulator
+  databaseURL: `${process.env.FIREBASE_DATABASE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`,
+  projectId: process.env.GCLOUD_PROJECT,
+});
+const firestore = app.firestore();
+const subcollection = "subcollection";
+const typesenseSubcollectionName = config.typesenseCollectionName + "/" + subcollection;
+
+describe("indexOnWriteWithoutFlattening", () => {
+  beforeEach(async () => {
+    // delete the Firestore collection
+    await firestore.recursiveDelete(firestore.collection(config.firestoreCollectionPath));
+    // delete the Typesense collections
+    const collectionsToDelete = [
+      typesenseSubcollectionName,
+      config.typesenseCollectionName,
+    ];
+    for (const collectionName of collectionsToDelete) {
+      try {
+        await typesense.collections(encodeURIComponent(collectionName)).delete();
+      } catch (e) {
+        console.info(`${collectionName} collection not found, proceeding...`);
+      }
+    }
+
+    // recreate the Typesense collection
+    await typesense.collections().create({
+      name: typesenseSubcollectionName,
+      fields: [{name: ".*", type: "auto"}],
+      enable_nested_fields: true,
+    });
+  });
+
+  afterAll(async () => {
+    // clean up the whole firebase app
+    await app.delete();
+  });
+  describe("Backfill from subcollections", () => {
+    it("backfills documents from subcollections to Typesense", async () => {
+      // Defne the parent document and subcollection data
+      process.env.FLATTEN_NESTED_DOCUMENTS = "false";
+
+      const docData = {
+        nested_field: {
+          field1: "value1",
+        },
+      };
+      const subDocData = {
+        nested_field: {
+          field1: "value1",
+          field2: ["value2", "value3", "value4"],
+          field3: {
+            fieldA: "valueA",
+            fieldB: ["valueB", "valueC", "valueD"],
+          },
+        },
+      };
+      // create parent document in Firestore
+      const docRef = await firestore.collection(config.firestoreCollectionPath).add(docData);
+
+      // create a subcollection with document under the parent document
+      const subCollectionRef = docRef.collection(subcollection);
+      const subDocRef = await subCollectionRef.add(subDocData);
+
+      // wait for the Firestore cloud function to write to Typesense
+      await new Promise((r) => setTimeout(r, 2500));
+
+      // check that the document was indexed
+      let typesenseDocsStr = await typesense
+        .collections(encodeURIComponent(typesenseSubcollectionName))
+        .documents()
+        .export();
+      let typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
+
+      expect(typesenseDocs.length).toBe(1);
+      expect(typesenseDocs[0]).toStrictEqual({
+        id: subDocData.id,
+        ...subDocData,
+      });
+
+      // update document in Firestore
+      subDocData.nested_field.field1 = "new value1";
+
+      await subDocRef.update(subDocData);
+
+      // wait for the Firestore cloud function to write to Typesense
+      await new Promise((r) => setTimeout(r, 2500));
+
+      // check that the document was updated
+      typesenseDocsStr = await typesense
+        .collections(encodeURIComponent(typesenseSubcollectionName))
+        .documents()
+        .export({exclude_fields: ""});
+      typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
+
+      expect(typesenseDocs.length).toBe(1);
+      expect(typesenseDocs[0]).toStrictEqual({
+        id: subDocData.id,
+        ...subDocData,
+      });
+
+      // delete both documents in Firestore
+      await subDocRef.delete();
+      await docRef.delete();
+
+      // wait for the Firestore cloud function to write to Typesense
+      await new Promise((r) => setTimeout(r, 2500));
+
+      // check that the subcollection document was deleted
+      typesenseDocsStr = await typesense.collections(encodeURIComponent(typesenseSubcollectionName)).documents().export();
+
+      expect(typesenseDocsStr).toBe("");
+    });
+  });
+});