diff --git a/lib/datastore.js b/lib/datastore.js index 4b978ad3..0aa269e9 100755 --- a/lib/datastore.js +++ b/lib/datastore.js @@ -134,7 +134,6 @@ Datastore.prototype.ensureIndex = function (options, cb) { this.indexes[options.fieldName] = new Index(options); if (options.expireAfterSeconds !== undefined) { this.ttlIndexes[options.fieldName] = options.expireAfterSeconds; } // With this implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here - try { this.indexes[options.fieldName].insert(this.getAllData()); } catch (e) { @@ -255,27 +254,44 @@ Datastore.prototype.updateIndexes = function (oldDoc, newDoc) { Datastore.prototype.getCandidates = function (query, dontExpireStaleDocs, callback) { var indexNames = Object.keys(this.indexes) , self = this - , usableQueryKeys; + , usableQueryKeys + , basicQueryKeys + , compoundQueryKeys; if (typeof dontExpireStaleDocs === 'function') { callback = dontExpireStaleDocs; dontExpireStaleDocs = false; } - async.waterfall([ // STEP 1: get candidates list by checking indexes from most to least frequent usecase function (cb) { - // For a basic match + usableQueryKeys = []; Object.keys(query).forEach(function (k) { if (typeof query[k] === 'string' || typeof query[k] === 'number' || typeof query[k] === 'boolean' || util.isDate(query[k]) || query[k] === null) { usableQueryKeys.push(k); } }); - usableQueryKeys = _.intersection(usableQueryKeys, indexNames); - if (usableQueryKeys.length > 0) { - return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]])); + + // For a basic match + basicQueryKeys = _.intersection(usableQueryKeys, indexNames); + if (basicQueryKeys.length > 0) { + return cb(null, self.indexes[basicQueryKeys[0]].getMatching(query[basicQueryKeys[0]])); + } + + // For a compound match + compoundQueryKeys = []; + indexNames.forEach(function(indexName){ + if (indexName.indexOf(',') === -1) return; + var subIndexNames = indexName.split(','); + if (_.intersection(subIndexNames, usableQueryKeys).length === subIndexNames.length) { + compoundQueryKeys.push(subIndexNames); + } + }); + + if (compoundQueryKeys.length > 0) { + return cb(null, self.indexes[compoundQueryKeys[0]].getMatching(_.pick(query,compoundQueryKeys[0]))); } // For a $in match diff --git a/lib/indexes.js b/lib/indexes.js index 941d98c5..18be875a 100755 --- a/lib/indexes.js +++ b/lib/indexes.js @@ -57,7 +57,6 @@ function Index (options) { this.sparse = options.sparse || false; var compareFunc = util.isArray(this.fieldName) ? model.compoundCompareThings(this.fieldName) : model.compareThings; - this.treeOptions = { unique: this.unique, compareKeys: compareFunc, checkValueEquality: checkValueEquality }; this.reset(); // No data in the beginning diff --git a/lib/persistence.js b/lib/persistence.js index 88a4948b..baecec30 100755 --- a/lib/persistence.js +++ b/lib/persistence.js @@ -129,8 +129,9 @@ Persistence.prototype.persistCachedDatabase = function (cb) { toPersist += self.afterSerialization(model.serialize(doc)) + '\n'; }); Object.keys(this.db.indexes).forEach(function (fieldName) { + if (fieldName != "_id") { // The special _id index is managed by datastore.js, the others need to be persisted - toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n'; + toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: self.db.indexes[fieldName].fieldName, unique: self.db.indexes[fieldName].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n'; } }); diff --git a/test/db.test.js b/test/db.test.js index ab68788b..3994c1d3 100755 --- a/test/db.test.js +++ b/test/db.test.js @@ -450,6 +450,28 @@ describe('Database', function () { }); }); + it('Can use a compound index to get docs with a basic match', function (done) { + d.ensureIndex({ fieldName: ['tf', 'tg'] }, function (err) { + d.insert({ tf: 4, tg: 0, foo: 1 }, function () { + d.insert({ tf: 6, tg: 0, foo: 2 }, function () { + d.insert({ tf: 4, tg: 1, foo: 3 }, function (err, _doc1) { + d.insert({ tf: 6, tg: 1, foo: 4 }, function () { + d.getCandidates({ tf: 4, tg: 1 }, function (err, data) { + var doc1 = _.find(data, function (d) { return d._id === _doc1._id; }) + ; + + data.length.should.equal(1); + assert.deepEqual(doc1, { _id: doc1._id, tf: 4, tg: 1, foo: 3 }); + + done(); + }); + }); + }); + }); + }); + }); + }); + it('Can use an index to get docs with a $in match', function (done) { d.ensureIndex({ fieldName: 'tf' }, function (err) { d.insert({ tf: 4 }, function (err) { diff --git a/test/indexes.test.js b/test/indexes.test.js index 876efdfd..b7f3c02b 100755 --- a/test/indexes.test.js +++ b/test/indexes.test.js @@ -34,6 +34,31 @@ describe('Indexes', function () { doc3.a.should.equal(42); }); + it('Can insert pointers to documents in the index correctly when they have compound fields', function () { + var idx = new Index({ fieldName: ['tf', 'tg'] }) + , doc1 = { a: 5, tf: 'hello', tg: 'world' } + , doc2 = { a: 8, tf: 'hello', tg: 'bloup' } + , doc3 = { a: 2, tf: 'bloup', tg: 'bloup' } + ; + + idx.insert(doc1); + idx.insert(doc2); + idx.insert(doc3); + + + // The underlying BST now has 3 nodes which contain the docs where it's expected + idx.tree.getNumberOfKeys().should.equal(3); + assert.deepEqual(idx.tree.search({tf: 'hello', tg: 'world'}), [{ a: 5, tf: 'hello', tg: 'world' }]); + assert.deepEqual(idx.tree.search({tf: 'hello', tg: 'bloup'}), [{ a: 8, tf: 'hello', tg: 'bloup' }]); + assert.deepEqual(idx.tree.search({tf: 'bloup', tg: 'bloup'}), [{ a: 2, tf: 'bloup', tg: 'bloup' }]); + + // The nodes contain pointers to the actual documents + idx.tree.search({tf: 'hello', tg: 'bloup'})[0].should.equal(doc2); + idx.tree.search({tf: 'bloup', tg: 'bloup'})[0].a = 42; + doc3.a.should.equal(42); + + }); + it('Inserting twice for the same fieldName in a unique index will result in an error thrown', function () { var idx = new Index({ fieldName: 'tf', unique: true }) , doc1 = { a: 5, tf: 'hello' } @@ -44,6 +69,16 @@ describe('Indexes', function () { (function () { idx.insert(doc1); }).should.throw(); }); + it('Inserting twice for the same compound fieldName in a unique index will result in an error thrown', function () { + var idx = new Index({ fieldName: ['tf', 'tg'], unique: true }) + , doc1 = { a: 5, tf: 'hello', tg: 'world' } + ; + + idx.insert(doc1); + idx.tree.getNumberOfKeys().should.equal(1); + (function () { idx.insert(doc1); }).should.throw(); + }); + it('Inserting twice for a fieldName the docs dont have with a unique index results in an error thrown', function () { var idx = new Index({ fieldName: 'nope', unique: true }) , doc1 = { a: 5, tf: 'hello' } diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 00000000..b40b46e7 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,172 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +assertion-error@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b" + integrity sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw== + +async@0.2.10: + version "0.2.10" + resolved "https://registry.yarnpkg.com/async/-/async-0.2.10.tgz#b6bbe0b0674b9d719708ca38de8c237cb526c3d1" + integrity sha1-trvgsGdLnXGXCMo43owjfLUmw9E= + +binary-search-tree@0.2.5: + version "0.2.5" + resolved "https://registry.yarnpkg.com/binary-search-tree/-/binary-search-tree-0.2.5.tgz#7dbb3b210fdca082450dad2334c304af39bdc784" + integrity sha1-fbs7IQ/coIJFDa0jNMMErzm9x4Q= + dependencies: + underscore "~1.4.4" + +chai@^3.2.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/chai/-/chai-3.5.0.tgz#4d02637b067fe958bdbfdd3a40ec56fef7373247" + integrity sha1-TQJjewZ/6Vi9v906QOxW/vc3Mkc= + dependencies: + assertion-error "^1.0.1" + deep-eql "^0.1.3" + type-detect "^1.0.0" + +commander@0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-0.6.1.tgz#fa68a14f6a945d54dbbe50d8cdb3320e9e3b1a06" + integrity sha1-+mihT2qUXVTbvlDYzbMyDp47GgY= + +commander@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-1.1.1.tgz#50d1651868ae60eccff0a2d9f34595376bc6b041" + integrity sha1-UNFlGGiuYOzP8KLZ80WVN2vGsEE= + dependencies: + keypress "0.1.x" + +debug@*: + version "4.3.1" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" + integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== + dependencies: + ms "2.1.2" + +deep-eql@^0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2" + integrity sha1-71WKyrjeJSBs1xOQbXTlaTDrafI= + dependencies: + type-detect "0.1.1" + +diff@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/diff/-/diff-1.0.2.tgz#4ae73f1aee8d6fcf484f1a1ce77ce651d9b7f0c9" + integrity sha1-Suc/Gu6Nb89ITxoc53zmUdm38Mk= + +exec-time@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/exec-time/-/exec-time-0.0.2.tgz#6331dc860cc5aa97a63d956e0fd847df8b15708c" + integrity sha1-YzHchgzFqpemPZVuD9hH34sVcIw= + +growl@1.5.x: + version "1.5.1" + resolved "https://registry.yarnpkg.com/growl/-/growl-1.5.1.tgz#1decd1f22a4b30dae7d363799ec624cf40cc0070" + integrity sha1-HezR8ipLMNrn02N5nsYkz0DMAHA= + +immediate@~3.0.5: + version "3.0.6" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps= + +jade@0.26.3: + version "0.26.3" + resolved "https://registry.yarnpkg.com/jade/-/jade-0.26.3.tgz#8f10d7977d8d79f2f6ff862a81b0513ccb25686c" + integrity sha1-jxDXl32NefL2/4YqgbBRPMslaGw= + dependencies: + commander "0.6.1" + mkdirp "0.3.0" + +keypress@0.1.x: + version "0.1.0" + resolved "https://registry.yarnpkg.com/keypress/-/keypress-0.1.0.tgz#4a3188d4291b66b4f65edb99f806aa9ae293592a" + integrity sha1-SjGI1CkbZrT2XtuZ+AaqmuKTWSo= + +lie@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e" + integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4= + dependencies: + immediate "~3.0.5" + +localforage@^1.3.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.9.0.tgz#f3e4d32a8300b362b4634cc4e066d9d00d2f09d1" + integrity sha512-rR1oyNrKulpe+VM9cYmcFn6tsHuokyVHFaCM3+osEmxaHTbEk8oQu6eGDfS6DQLWi/N67XRmB8ECG37OES368g== + dependencies: + lie "3.1.1" + +minimist@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" + integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + +mkdirp@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.0.tgz#1bbf5ab1ba827af23575143490426455f481fe1e" + integrity sha1-G79asbqCevI1dRQ0kEJkVfSB/h4= + +mkdirp@0.3.3: + version "0.3.3" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.3.tgz#595e251c1370c3a68bab2136d0e348b8105adf13" + integrity sha1-WV4lHBNww6aLqyE20ONIuBBa3xM= + +mkdirp@~0.5.1: + version "0.5.5" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" + integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== + dependencies: + minimist "^1.2.5" + +mocha@1.4.x: + version "1.4.3" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-1.4.3.tgz#f36630651e0686fe283d1bcac820b52acc091407" + integrity sha1-82YwZR4Ghv4oPRvKyCC1KswJFAc= + dependencies: + commander "0.6.1" + debug "*" + diff "1.0.2" + growl "1.5.x" + jade "0.26.3" + mkdirp "0.3.3" + ms "0.3.0" + +ms@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.3.0.tgz#03edc348d613e66a56486cfdac53bcbe899cbd61" + integrity sha1-A+3DSNYT5mpWSGz9rFO8vomcvWE= + +ms@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + +request@2.9.x: + version "2.9.203" + resolved "https://registry.yarnpkg.com/request/-/request-2.9.203.tgz#6c1711a5407fb94a114219563e44145bcbf4723a" + integrity sha1-bBcRpUB/uUoRQhlWPkQUW8v0cjo= + +sinon@1.3.x: + version "1.3.4" + resolved "https://registry.yarnpkg.com/sinon/-/sinon-1.3.4.tgz#9fe5d746acb5e078f26f6598a18b13cfa066c4ab" + integrity sha1-n+XXRqy14Hjyb2WYoYsTz6BmxKs= + +type-detect@0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-0.1.1.tgz#0ba5ec2a885640e470ea4e8505971900dac58822" + integrity sha1-C6XsKohWQORw6k6FBZcZANrFiCI= + +type-detect@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" + integrity sha1-diIXzAbbJY7EiQihKY6LlRIejqI= + +underscore@~1.4.4: + version "1.4.4" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604" + integrity sha1-YaajIBBiKvoHljvzJSA88SI51gQ=