Merge pull request #91 from tharropoulos/dot-notation

feat(utils): implement nested field extraction

functions/package.json

@@ -18,7 +18,7 @@
     "@babel/runtime": "^7.24.7",
     "firebase-admin": "^12.2.0",
     "firebase-functions": "^5.0.1",
-    "flat": "^6.0.1",
+    "lodash.get": "^4.4.2",
     "typesense": "^1.8.2"
   },
   "devDependencies": {

functions/src/utils.js

@@ -27,27 +27,124 @@ const mapValue = (value) => {
   }
 };
 
+/**
+ * Sets a nested value in an object using a dot-notated path.
+ * @param {Object} obj - The object to modify.
+ * @param {string} path - The dot-notated path to the value.
+ * @param {*} value - The value to set.
+ * @return {Object} The modified object.
+ */
+function setNestedValue(obj, path, value) {
+  const keys = path.split(".");
+  keys.reduce((acc, key, index) => {
+    if (index === keys.length - 1) {
+      acc[key] = value;
+      return acc;
+    }
+    if (acc[key] === undefined) {
+      acc[key] = Number.isInteger(+keys[index + 1]) ? [] : {};
+    }
+    return acc[key];
+  }, obj);
+  return obj;
+}
+
+/**
+ * Gets a nested value from an object using a dot-notated path.
+ * @param {Object} obj - The object to retrieve the value from.
+ * @param {string} path - The dot-notated path to the value.
+ * @return {*} The value at the specified path, or undefined if not found.
+ */
+function getNestedValue(obj, path) {
+  const keys = path.split(".");
+  return keys.reduce((current, key) => {
+    if (current === undefined) return undefined;
+    if (Array.isArray(current)) {
+      return Number.isInteger(+key) ? current[+key] : current.map((item) => ({[key]: item[key]}));
+    }
+    return current[key];
+  }, obj);
+}
+
+/**
+ * Merges an array of objects into a single array, combining objects at the same index.
+ * @param {Array<Object[]>} arrays - An array of object arrays to merge.
+ * @return {Object[]} A merged array of objects.
+ */
+function mergeArrays(arrays) {
+  const maxLength = Math.max(...arrays.map((arr) => arr.length));
+  return Array.from({length: maxLength}, (_, i) => Object.assign({}, ...arrays.map((arr) => arr[i] || {})));
+}
+
+/**
+ * Extracts a field from the data and adds it to the accumulator.
+ * @param {Object} data - The source data object.
+ * @param {Object} acc - The accumulator object.
+ * @param {string} field - The field to extract.
+ * @return {Object} The updated accumulator.
+ */
+function extractField(data, acc, field) {
+  const value = getNestedValue(data, field);
+  if (value === undefined) return acc;
+  const [topLevelField] = field.split(".");
+  const isArrayOfObjects = Array.isArray(value) && typeof value[0] === "object";
+  if (isArrayOfObjects) {
+    return {
+      ...acc,
+      [topLevelField]: acc[topLevelField] ? mergeArrays([acc[topLevelField], value]) : value,
+    };
+  } else {
+    return setNestedValue(acc, field, value);
+  }
+}
+
+/**
+ * Flattens a nested object, converting nested properties to dot-notation.
+ * @param {Object} obj - The object to flatten.
+ * @param {string} [prefix=""] - The prefix to use for flattened keys.
+ * @return {Object} A new flattened object.
+ */
+function flattenDocument(obj, prefix = "") {
+  return Object.keys(obj).reduce((acc, key) => {
+    const newKey = prefix ? `${prefix}.${key}` : key;
+    const value = obj[key];
+    // Handle primitive values (including null)
+    if (typeof value !== "object" || value === null) {
+      acc[newKey] = value;
+      return acc;
+    }
+    // Handle arrays
+    if (Array.isArray(value)) {
+      if (value.length === 0 || typeof value[0] !== "object") {
+        acc[newKey] = value;
+        return acc;
+      }
+      Object.keys(value[0]).forEach((subKey) => {
+        acc[`${newKey}.${subKey}`] = value.map((item) => item[subKey]).filter((v) => v !== undefined);
+      });
+      return acc;
+    }
+    // Handle nested objects
+    return {...acc, ...flattenDocument(value, newKey)};
+  }, {});
+}
+
 /**
  * @param {DocumentSnapshot} firestoreDocumentSnapshot
  * @param {Array} fieldsToExtract
  * @return {Object} typesenseDocument
  */
 exports.typesenseDocumentFromSnapshot = async (firestoreDocumentSnapshot, fieldsToExtract = config.firestoreCollectionFields) => {
-  const flat = await import("flat");
   const data = firestoreDocumentSnapshot.data();
 
-  let entries = Object.entries(data);
-
-  if (fieldsToExtract.length) {
-    entries = entries.filter(([key]) => fieldsToExtract.includes(key));
-  }
+  const extractedData = fieldsToExtract.length === 0 ? data : fieldsToExtract.reduce((acc, field) => extractField(data, acc, field), {});
 
-  // Build a document with just the fields requested by the user, and mapped from Firestore types to Typesense types
-  const mappedDocument = Object.fromEntries(entries.map(([key, value]) => [key, mapValue(value)]));
+  const mappedDocument = Object.fromEntries(Object.entries(extractedData).map(([key, value]) => [key, mapValue(value)]));
 
   // using flat to flatten nested objects for older versions of Typesense that did not support nested fields
   // https://typesense.org/docs/0.22.2/api/collections.html#indexing-nested-fields
-  const typesenseDocument = config.shouldFlattenNestedDocuments ? flat.flatten(mappedDocument, {safe: true}) : mappedDocument;
+  const typesenseDocument = config.shouldFlattenNestedDocuments ? flattenDocument(mappedDocument) : mappedDocument;
+  console.log("typesenseDocument", typesenseDocument);
 
   typesenseDocument.id = firestoreDocumentSnapshot.id;
 

package.json

@@ -5,8 +5,8 @@
     "emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data",
     "export": "firebase emulators:export emulator_data",
     "test": "npm run test-part-1 && npm run test-part-2",
-    "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=test/indexOnWriteWithoutFlattening.spec.js'",
-    "test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathPattern=test/indexOnWriteWithoutFlattening.spec.js'",
+    "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=\"WithoutFlattening\"'",
+    "test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'",
     "typesenseServer": "docker compose up",
     "lint:fix": "eslint . --fix",
     "lint": "eslint ."

test/utils.spec.js → test/utilsWithFlattening.spec.js

@@ -117,5 +117,87 @@ describe("Utils", () => {
         });
       });
     });
+    describe("Nested fields extraction", () => {
+      it("extracts nested fields using dot notation", async () => {
+        const typesenseDocumentFromSnapshot = (await import("../functions/src/utils.js")).typesenseDocumentFromSnapshot;
+        const documentSnapshot = test.firestore.makeDocumentSnapshot(
+          {
+            user: {
+              name: "John Doe",
+              address: {
+                city: "New York",
+                country: "USA",
+              },
+            },
+            tags: ["tag1", "tag2"],
+          },
+          "id",
+        );
+        const result = await typesenseDocumentFromSnapshot(documentSnapshot, ["user.name", "user.address.city", "tags"]);
+        expect(result).toEqual({
+          id: "id",
+          "user.name": "John Doe",
+          "user.address.city": "New York",
+          tags: ["tag1", "tag2"],
+        });
+      });
+
+      it("handles missing nested fields gracefully", async () => {
+        const typesenseDocumentFromSnapshot = (await import("../functions/src/utils.js")).typesenseDocumentFromSnapshot;
+        const documentSnapshot = test.firestore.makeDocumentSnapshot(
+          {
+            user: {
+              name: "John Doe",
+            },
+          },
+          "id",
+        );
+        const result = await typesenseDocumentFromSnapshot(documentSnapshot, ["user.name", "user.address.city"]);
+        expect(result).toEqual({
+          id: "id",
+          "user.name": "John Doe",
+        });
+      });
+
+      it("extracts nested fields alongside top-level fields", async () => {
+        const typesenseDocumentFromSnapshot = (await import("../functions/src/utils.js")).typesenseDocumentFromSnapshot;
+        const documentSnapshot = test.firestore.makeDocumentSnapshot(
+          {
+            title: "Main Title",
+            user: {
+              name: "John Doe",
+              age: 30,
+            },
+          },
+          "id",
+        );
+        const result = await typesenseDocumentFromSnapshot(documentSnapshot, ["title", "user.name"]);
+        expect(result).toEqual({
+          id: "id",
+          title: "Main Title",
+          "user.name": "John Doe",
+        });
+      });
+
+      it("handles array indexing in dot notation", async () => {
+        const typesenseDocumentFromSnapshot = (await import("../functions/src/utils.js")).typesenseDocumentFromSnapshot;
+        const documentSnapshot = test.firestore.makeDocumentSnapshot(
+          {
+            comments: [
+              {author: "Alice", text: "Great post!"},
+              {author: "Bob", text: "Thanks for sharing.", likes: 5},
+            ],
+          },
+          "id",
+        );
+        const result = await typesenseDocumentFromSnapshot(documentSnapshot, ["comments.author", "comments.text", "comments.likes"]);
+        expect(result).toEqual({
+          id: "id",
+          "comments.author": ["Alice", "Bob"],
+          "comments.text": ["Great post!", "Thanks for sharing."],
+          "comments.likes": [5],
+        });
+      });
+    });
   });
 });