feat: adds @orama/plugin-pt15 (#817)

benchmarks/algorithms.js

@@ -0,0 +1,42 @@
+import b from 'benny'
+import { create, insertMultiple, search } from 'orama_latest'
+import { pluginPT15 } from '@orama/plugin-pt15'
+import dataset from './src/dataset.json' assert { type: 'json' }
+import {stopwords} from '@orama/stopwords/english'
+
+const dbBM25 = create({
+    schema: {
+        description: 'string'
+    },
+    components: {
+        tokenizer: {
+            stopWords: stopwords,
+        },
+    }
+})
+const dbWithPT15 = create({
+    schema: {
+        description: 'string'
+    },
+    plugins: [pluginPT15()],
+    components: {
+        tokenizer: {
+            stopWords: stopwords,
+        },
+    }
+})
+await insertMultiple(dbBM25, dataset)
+await insertMultiple(dbWithPT15, dataset)
+
+b.suite('search-algorithms',
+    b.add('search bm25', () => {
+        search(dbBM25, { term: 'L' })
+    }),
+    b.add('search pt15', () => {
+        search(dbWithPT15, { term: 'L' })
+    }),
+    b.cycle(),
+    b.complete(),
+    b.save({ file: 'insert', version: '1.0.0' }),
+    b.save({ file: 'search-algorithms', format: 'chart.html' }),
+)

benchmarks/index.js

@@ -12,6 +12,9 @@ function benchmarkInsert() {
     b.add('insert in Orama latest', () => {
       insert.oramaLatest()
     }),
+    b.add('insert in Orama latest with PT15', () => {
+      insert.oramaLatestPT15()
+    }),
     b.cycle(),
     b.complete(),
     b.save({ file: 'insert', version: '1.0.0' }),
@@ -30,6 +33,9 @@ function benchmarkInsertMultiple() {
     b.add('insert multiple in Orama latest', () => {
       insertMultiple.oramaLatest()
     }),
+    b.add('insert multiple in Orama latest with PT15', () => {
+      insertMultiple.oramaLatestPT15()
+    }),
     b.cycle(),
     b.complete(),
     b.save({ file: 'insert multiple', version: '1.0.0' }),
@@ -48,6 +54,9 @@ function benchmarkSearch() {
     b.add('plain search in Orama latest', () => {
       searchPlain.oramaLatest()
     }),
+    b.add('plain search in Orama latest with PT15', () => {
+      searchPlain.oramaLatestPT15()
+    }),
     b.cycle(),
     b.complete(),
     b.save({ file: 'plain search', version: '1.0.0' }),
@@ -66,6 +75,9 @@ function benchmarkSearchWithFilters() {
     b.add('search with filters in Orama latest', () => {
       searchWithFilters.oramaLatest()
     }),
+    b.add('search with filters in Orama latest with PT15', () => {
+      searchWithFilters.oramaLatestPT15()
+    }),
     b.cycle(),
     b.complete(),
     b.save({ file: 'search with filters', version: '1.0.0' }),
@@ -84,6 +96,9 @@ function benchmarkSearchWithLongTextAndComplexFilters() {
     b.add('search with long text and complex filters in Orama latest', () => {
       searchWithLongTextAndComplexFilters.oramaLatest()
     }),
+    b.add('search with long text and complex filters in Orama latest with PT15', () => {
+      searchWithLongTextAndComplexFilters.oramaLatestPT15()
+    }),
     b.cycle(),
     b.complete(),
     b.save({ file: 'search with long text and complex filters', version: '1.0.0' }),

benchmarks/package.json

@@ -13,7 +13,9 @@
   "dependencies": {
     "orama_211": "npm:@orama/[email protected]",
     "orama_300_rc_2": "npm:@orama/[email protected]",
-    "orama_latest": "file:../packages/orama"
+    "orama_latest": "file:../packages/orama",
+    "@orama/stopwords": "file:../packages/stopwords",
+    "@orama/plugin-pt15": "file:../packages/plugin-pt15"
   },
   "devDependencies": {
     "benny": "^3.7.1"

benchmarks/src/get-orama.js

@@ -1,6 +1,7 @@
 import * as orama211 from 'orama_211'
 import * as orama300rc2 from 'orama_300_rc_2'
 import * as oramaLatest from 'orama_latest'
+import { pluginPT15 } from '@orama/plugin-pt15'
 import dataset from './dataset.json' assert { type: 'json' }
 
 export const schema = {
@@ -13,12 +14,14 @@ export const schema = {
 const create = {
   orama211: () => orama211.create({ schema }),
   orama300rc2: () => orama300rc2.create({ schema }),
-  oramaLatest: () => oramaLatest.create({ schema })
+  oramaLatest: () => oramaLatest.create({ schema }),
+  oramaLatestPT15: () => oramaLatest.create({ schema, plugins: [pluginPT15()] })
 }
 
 const db211 = await create.orama211()
 const db300rc2 = create.orama300rc2()
 const dbLatest = create.oramaLatest()
+const dbLatestPT15 = create.oramaLatestPT15()
 
 export const insert = {
   orama211: async () => {
@@ -38,6 +41,12 @@ export const insert = {
     for (const record of dataset) {
       oramaLatest.insert(db, record)
     }
+  },
+  oramaLatestPT15: () => {
+    const db = create.oramaLatestPT15()
+    for (const record of dataset) {
+      oramaLatest.insert(db, record)
+    }
   }
 }
 
@@ -50,6 +59,9 @@ export const insertMultiple = {
   },
   oramaLatest: () => {
     oramaLatest.insertMultiple(dbLatest, dataset, 50)
+  },
+  oramaLatestPT15: () => {
+    oramaLatest.insertMultiple(dbLatestPT15, dataset, 50)
   }
 }
 
@@ -62,6 +74,9 @@ export const searchPlain = {
   },
   oramaLatest: () => {
     oramaLatest.search(dbLatest, { term: 'Legend of Zelda' })
+  },
+  oramaLatestPT15: () => {
+    oramaLatest.search(dbLatestPT15, { term: 'Legend of Zelda' })
   }
 }
 
@@ -74,6 +89,9 @@ export const searchWithFilters = {
   },
   oramaLatest: () => {
     oramaLatest.search(dbLatest, { term: 'Super Hero', where: { rating: { gte: 4 } } })
+  },
+  oramaLatestPT15: () => {
+    oramaLatest.search(dbLatestPT15, { term: 'Super Hero', where: { rating: { gte: 4 } } })
   }
 }
 
@@ -86,5 +104,8 @@ export const searchWithLongTextAndComplexFilters = {
   },
   oramaLatest: () => {
     oramaLatest.search(dbLatest, { term: 'classic run gun, action game focused on boss battles', where: { rating: { gte: 4 }, genres: { containsAll: ['Shooter'] } } })
+  },
+  oramaLatestPT15: () => {
+    oramaLatest.search(dbLatestPT15, { term: 'classic run gun, action game focused on boss battles', where: { rating: { gte: 4 }, genres: { containsAll: ['Shooter'] } } })
   }
 }

packages/orama/src/components/documents-store.ts

@@ -51,9 +51,7 @@ export function getAll<T extends AnyOrama, ResultDocument extends TypedDocument<
   return store.docs
 }
 
-export function store(store: DocumentsStore, id: DocumentID, doc: AnyDocument): boolean {
-  const internalId = getInternalDocumentId(store.sharedInternalDocumentStore, id)
-
+export function store(store: DocumentsStore, id: DocumentID, internalId: InternalDocumentID, doc: AnyDocument): boolean {
   if (typeof store.docs[internalId] !== 'undefined') {
     return false
   }

packages/orama/src/components/index.ts

@@ -287,7 +287,7 @@ export function insert(
   }
 }
 
-function insertVector(index: Index, prop: string, value: number[] | VectorType, id: DocumentID): void {
+export function insertVector(index: AnyIndexStore, prop: string, value: number[] | VectorType, id: DocumentID): void {
   if (!(value instanceof Float32Array)) {
     value = new Float32Array(value)
   }
@@ -303,14 +303,13 @@ function removeScalar(
   index: Index,
   prop: string,
   id: DocumentID,
+  internalId: InternalDocumentID,
   value: SearchableValue,
   schemaType: ScalarSearchableType,
   language: string | undefined,
   tokenizer: Tokenizer,
   docsCount: number
 ): boolean {
-  const internalId = getInternalDocumentId(index.sharedInternalDocumentStore, id)
-
   if (isVectorType(schemaType)) {
     delete index.vectorIndexes[prop].vectors[id]
     return true
@@ -354,6 +353,7 @@ export function remove(
   index: Index,
   prop: string,
   id: DocumentID,
+  internalId: InternalDocumentID,
   value: SearchableValue,
   schemaType: SearchableType,
   language: string | undefined,
@@ -366,6 +366,7 @@ export function remove(
       index,
       prop,
       id,
+      internalId,
       value,
       schemaType as ScalarSearchableType,
       language,
@@ -379,7 +380,7 @@ export function remove(
   const elements = value as Array<string | number | boolean>
   const elementsLength = elements.length
   for (let i = 0; i < elementsLength; i++) {
-    removeScalar(implementation, index, prop, id, elements[i], innerSchemaType, language, tokenizer, docsCount)
+    removeScalar(implementation, index, prop, id, internalId, elements[i], innerSchemaType, language, tokenizer, docsCount)
   }
 
   return true

packages/orama/src/errors.ts

@@ -42,7 +42,8 @@ const errors = {
   PLUGIN_CRASHED: `A plugin crashed during initialization. Please check the error message for more information:`,
   PLUGIN_SECURE_PROXY_NOT_FOUND: `Could not find '@orama/secure-proxy-plugin' installed in your Orama instance.\nPlease install it before proceeding with creating an answer session.\nRead more at https://docs.orama.com/open-source/plugins/plugin-secure-proxy\n`,
   PLUGIN_SECURE_PROXY_MISSING_CHAT_MODEL: `Could not find a chat model defined in the secure proxy plugin configuration.\nPlease provide a chat model before proceeding with creating an answer session.\nRead more at https://docs.orama.com/open-source/plugins/plugin-secure-proxy\n`,
-  ANSWER_SESSION_LAST_MESSAGE_IS_NOT_ASSISTANT: `The last message in the session is not an assistant message. Cannot regenerate non-assistant messages.`
+  ANSWER_SESSION_LAST_MESSAGE_IS_NOT_ASSISTANT: `The last message in the session is not an assistant message. Cannot regenerate non-assistant messages.`,
+  PLUGIN_COMPONENT_CONFLICT: `The component "%s" is already defined. The plugin "%s" is trying to redefine it.`,
 }
 
 export type ErrorCode = keyof typeof errors