add document title to chroma metadata

Author: valeksiev

package.json

@@ -55,6 +55,6 @@
     "npm": ">=8.5.5"
   },
   "volta": {
-    "node": "16.15.0"
+    "node": "21.7.3"
   }
 }

src/documents.ts

@@ -11,11 +11,12 @@ export default class Documents {
     id: string,
     document: string,
     source: string,
-    type: DocumentType
+    type: DocumentType,
+    title: string
   ): Documents {
     this.ids.push(id);
     this.documents.push(document);
-    this.metadatas.push({ source, type });
+    this.metadatas.push({ source, type, title });
     return this;
   }
 

src/generate-document.ts

@@ -5,6 +5,7 @@ interface GeneratedDocument {
   source: string;
   document: string;
   type: DocumentType;
+  title: string;
 }
 
 export default (docLike: any): GeneratedDocument => {
@@ -50,5 +51,5 @@ export default (docLike: any): GeneratedDocument => {
   if (processedRefs) document = `${document}\nReferences:\n${processedRefs}`;
 
   document = `${document}\nURL: ${source}`;
-  return { id, source, type, document };
+  return { id, source, type, document, title: displayName };
 };

src/index.ts

@@ -16,19 +16,19 @@ export const main = async (spaceId: string) => {
   logger.info(`Space ${space.nameID} loaded.`);
 
   const documents = new Documents();
-  const { id, source, document, type } = generateDocument(space);
-  documents.add(id, document, source, type);
+  const { id, source, document, type, title } = generateDocument(space);
+  documents.add(id, document, source, type, title);
 
   for (let i = 0; i < (space.subspaces || []).length; i++) {
     const challenge = (space.subspaces || [])[i];
-    const { id, source, document, type } = generateDocument(challenge);
-    documents.add(id, document, source, type);
+    const { id, source, document, type, title } = generateDocument(challenge);
+    documents.add(id, document, source, type, title);
 
     for (let j = 0; j < (challenge.collaboration?.callouts || []).length; j++) {
       const callout = (challenge.collaboration?.callouts || [])[j];
       const { id, type } = callout;
       const generated = generateDocument(callout.framing);
-      const source = generated.source;
+      const { title, source } = generated;
       let document = generated.document;
 
       // extra loop but will do for now
@@ -59,7 +59,13 @@ export const main = async (spaceId: string) => {
       if (processedMessages)
         document = `${document}\nMessages:\n${processedMessages}`;
 
-      documents.add(id, document, source, type as unknown as DocumentType);
+      documents.add(
+        id,
+        document,
+        source,
+        type as unknown as DocumentType,
+        title
+      );
     }
   }
   await ingest(space.nameID, documents);

src/ingest.ts

@@ -29,7 +29,7 @@ export default async (space: string, docs: Documents) => {
 
   const collection = await client.getOrCreateCollection({ name: space });
 
-  await collection.add({
+  await collection.upsert({
     ...forEmbed,
     embeddings: data.map(({ embedding }) => embedding),
   });