Add indexers that allow to create ad-hoc indexes

src/Storage/WritableStorage.js

@@ -57,6 +57,7 @@ class WritableStorage extends ReadableStorage {
         }
         super(storageName, config);
         this.partitioner = config.partitioner;
+        this.indexers = [];
     }
 
     /**
@@ -188,6 +189,7 @@ class WritableStorage extends ReadableStorage {
         assert(position !== false, 'Error writing document.');
 
         const indexEntry = this.addIndex(partition.id, position, dataSize, document);
+        this.runIndexers(document);
         this.forEachSecondaryIndex((index, name) => {
             if (!index.isOpen()) {
                 index.open();
@@ -199,17 +201,28 @@ class WritableStorage extends ReadableStorage {
         return this.index.length;
     }
 
+    /**
+     * Add an indexer, which will be invoked for every document and ensures an index exists with the returned name and matcher.
+     * @param {function(document): {name:string, matcher:object|function}|null} indexer The indexer function, which returns an object containing the index name and matcher.
+     */
+    addIndexer(indexer) {
+        if (typeof indexer === 'function') {
+            this.indexers.push(indexer);
+        }
+    }
+
     /**
      * Ensure that an index with the given name and document matcher exists.
      * Will create the index if it doesn't exist, otherwise return the existing index.
      *
      * @api
      * @param {string} name The index name.
      * @param {object|function} [matcher] An object that describes the document properties that need to match to add it this index or a function that receives a document and returns true if the document should be indexed.
-     * @returns {ReadableIndex} The index containing all documents that match the query.
+     * @param {boolean} [updateIndex] If set to false the index will not be matched against all existing documents.
+     * @returns {ReadableIndex|WritableIndex} The index containing all documents that match the query.
      * @throws {Error} if the index doesn't exist yet and no matcher was specified.
      */
-    ensureIndex(name, matcher) {
+    ensureIndex(name, matcher, updateIndex = true) {
         if (name in this.secondaryIndexes) {
             return this.secondaryIndexes[name].index;
         }
@@ -223,6 +236,24 @@ class WritableStorage extends ReadableStorage {
 
         const metadata = buildMetadataForMatcher(matcher, this.hmac);
         const { index } = this.createIndex(indexName, Object.assign({}, this.indexOptions, { metadata }));
+        if (updateIndex) {
+            this.updateIndex(index, matcher);
+        }
+
+        this.secondaryIndexes[name] = { index, matcher };
+        this.emit('index-created', name);
+        return index;
+    }
+
+    /**
+     * Run the given matcher through all existing documents and add them to the index on match.
+     * If an error occurs during indexing, the index will be destroyed.
+     *
+     * @private
+     * @param {WritableIndex} index
+     * @param {object|function} matcher
+     */
+    updateIndex(index, matcher) {
         try {
             this.forEachDocument((document, indexEntry) => {
                 if (matches(document, matcher)) {
@@ -233,10 +264,6 @@ class WritableStorage extends ReadableStorage {
             index.destroy();
             throw e;
         }
-
-        this.secondaryIndexes[name] = { index, matcher };
-        this.emit('index-created', name);
-        return index;
     }
 
     /**
@@ -273,6 +300,23 @@ class WritableStorage extends ReadableStorage {
         }
     }
 
+    /**
+     * Run all indexers against the given document and ensure according indexes.
+     * Note: This will not cause newly created indexes to index all existing documents.
+     *
+     * @private
+     * @param {object} document
+     */
+    runIndexers(document) {
+        for (let indexer of this.indexers) {
+            const index = indexer(document);
+            if (!index || !index.name) {
+                continue;
+            }
+            this.ensureIndex(index.name, index.matcher, false);
+        }
+    }
+
     /**
      * Truncate all partitions after the given (global) sequence number.
      *

test/Storage.spec.js

@@ -121,6 +121,27 @@ describe('Storage', function() {
             expect(index.isOpen()).to.be(true);
         });
 
+        it('invokes indexers and correctly indexes documents', function() {
+            const sampleIndexer = (document) => {
+                const name = 'type-' + document.type;
+                const matcher = { type: document.type };
+                return {name, matcher};
+            };
+            storage = createStorage();
+            storage.addIndexer(sampleIndexer);
+
+            expect(() => storage.openIndex('type-bar')).to.throwError();
+            expect(() => storage.openIndex('type-foo')).to.throwError();
+            for (let i = 1; i <= 10; i++) {
+                storage.write({ type: (i % 3) ? 'bar' : 'foo', id: i });
+            }
+            const barIndex = storage.openIndex('type-bar');
+            const fooIndex = storage.openIndex('type-foo');
+
+            expect(barIndex.length).to.be(7);
+            expect(fooIndex.length).to.be(3);
+        });
+
     });
 
     describe('length', function() {