From 654cd4c68d806392ea232db622ee6b833581ac41 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Fri, 30 Nov 2018 17:21:39 +0100 Subject: [PATCH 01/27] feat: implementation of the new `resolve()` function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: `resolve()` replaces parts of `get()`. The API docs for it: > Retrieves IPLD Nodes along the `path` that is rooted at `cid`. - `cid` (`CID`, required): the CID the resolving starts. - `path` (`IPLD Path`, required): the path that should be resolved. Returns an async iterator of all the IPLD Nodes that were traversed during the path resolving. Every element is an object with these fields: - `remainderPath` (`string`): the part of the path that wasn’t resolved yet. - `value` (`*`): the value where the resolved path points to. If further traversing is possible, then the value is a CID object linking to another IPLD Node. If it was possible to fully resolve the path, `value` is the value the `path` points to. So if you need the CID of the IPLD Node you’re currently at, just take the `value` of the previously returned IPLD Node. --- package.json | 1 + src/index.js | 156 ++++++++++------------------- src/util.js | 32 ++++++ test/basics.js | 35 +++---- test/format-support.js | 40 +++----- test/ipld-all.js | 16 +-- test/ipld-bitcoin.js | 130 +++++++++++++----------- test/ipld-dag-cbor.js | 221 ++++++++++++++++------------------------- test/ipld-dag-pb.js | 163 ++++++++++++++---------------- test/ipld-eth-block.js | 138 +++++++++++++------------ test/ipld-eth.js | 31 +++--- test/ipld-git.js | 165 +++++++++++++++++------------- test/ipld-zcash.js | 130 +++++++++++++----------- 13 files changed, 616 insertions(+), 642 deletions(-) create mode 100644 src/util.js diff --git a/package.json b/package.json index 1837b6c..fd5f7ba 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "aegir": "^18.2.1", "bitcoinjs-lib": "^4.0.2", "chai": "^4.2.0", + "chai-as-promised": "^7.1.1", "dirty-chai": "^2.0.1", "ethereumjs-block": "^2.1.0", "ipfs-block-service": "~0.15.2", diff --git a/src/index.js b/src/index.js index 69dc5a6..c0fc840 100644 --- a/src/index.js +++ b/src/index.js @@ -3,9 +3,6 @@ const Block = require('ipfs-block') const pull = require('pull-stream') const CID = require('cids') -const doUntil = require('async/doUntil') -const joinPath = require('path').join -const osPathSep = require('path').sep const pullDeferSource = require('pull-defer').source const pullTraverse = require('pull-traverse') const map = require('async/map') @@ -14,6 +11,7 @@ const mergeOptions = require('merge-options') const ipldDagCbor = require('ipld-dag-cbor') const ipldDagPb = require('ipld-dag-pb') const ipldRaw = require('ipld-raw') +const { fancyIterator } = require('./util') function noop () {} @@ -62,111 +60,76 @@ class IPLDResolver { } } - get (cid, path, options, callback) { - if (typeof path === 'function') { - callback = path - path = undefined - } - - if (typeof options === 'function') { - callback = options - options = {} + /** + * Retrieves IPLD Nodes along the `path` that is rooted at `cid`. + * + * @param {CID} cid - the CID the resolving starts. + * @param {string} path - the path that should be resolved. + * @returns {Iterable.>} - Returns an async iterator of all the IPLD Nodes that were traversed during the path resolving. Every element is an object with these fields: + * - `remainderPath`: the part of the path that wasn’t resolved yet. + * - `value`: the value where the resolved path points to. If further traversing is possible, then the value is a CID object linking to another IPLD Node. If it was possible to fully resolve the path, value is the value the path points to. So if you need the CID of the IPLD Node you’re currently at, just take the value of the previously returned IPLD Node. + */ + resolve (cid, path) { + if (!CID.isCID(cid)) { + throw new Error('`cid` argument must be a CID') } - - // this removes occurrences of ./, //, ../ - // makes sure that path never starts with ./ or / - // path.join is OS specific. Need to convert back to POSIX format. - if (typeof path === 'string') { - path = joinPath('/', path) - .substr(1) - .split(osPathSep) - .join('/') + if (typeof path !== 'string') { + throw new Error('`path` argument must be a string') } - if (path === '' || !path) { - return this._get(cid, (err, node) => { - if (err) { - return callback(err) - } - callback(null, { - value: node, - remainderPath: '', - cid - }) - }) - } - - let value + const next = () => { + // End iteration if there isn't a CID to follow anymore + if (cid === null) { + return Promise.resolve({ done: true }) + } - doUntil( - (cb) => { + return new Promise((resolve, reject) => { this._getFormat(cid.codec, (err, format) => { - if (err) return cb(err) + if (err) { + return reject(err) + } // get block // use local resolver // update path value this.bs.get(cid, (err, block) => { if (err) { - return cb(err) + return reject(err) } format.resolver.resolve(block.data, path, (err, result) => { if (err) { - return cb(err) + return reject(err) } - value = result.value + + // Prepare for the next iteration if there is a `remainderPath` path = result.remainderPath - cb() + let value = result.value + // NOTE vmx 2018-11-29: Not all IPLD Formats return links as + // CIDs yet. Hence try to convert old style links to CIDs + if (Object.keys(value).length === 1 && '/' in value) { + value = new CID(value['/']) + } + if (CID.isCID(value)) { + cid = value + } else { + cid = null + } + + return resolve({ + done: false, + value: { + remainderPath: path, + value + } + }) }) }) }) - }, - () => { - const endReached = !path || path === '' || path === '/' - const isTerminal = value && !IPLDResolver._maybeCID(value) - - if ((endReached && isTerminal) || options.localResolve) { - cid = IPLDResolver._maybeCID(value) || cid - - return true - } else { - value = IPLDResolver._maybeCID(value) - // continue traversing - if (value) { - cid = value - } - return false - } - }, - (err, results) => { - if (err) { - return callback(err) - } - return callback(null, { - value: value, - remainderPath: path, - cid - }) - } - ) - } - - getStream (cid, path, options) { - const deferred = pullDeferSource() - - this.get(cid, path, options, (err, result) => { - if (err) { - return deferred.resolve( - pull.error(err) - ) - } - deferred.resolve( - pull.values([result]) - ) - }) + }) + } - return deferred + return fancyIterator(next) } /** @@ -347,25 +310,6 @@ class IPLDResolver { /* */ /* internals */ /* */ - - _get (cid, callback) { - waterfall([ - (cb) => this._getFormat(cid.codec, cb), - (format, cb) => this.bs.get(cid, (err, block) => { - if (err) return cb(err) - cb(null, format, block) - }), - (format, block, cb) => { - format.util.deserialize(block.data, (err, deserialized) => { - if (err) { - return cb(err) - } - cb(null, deserialized) - }) - } - ], callback) - } - _getFormat (codec, callback) { if (this.resolvers[codec]) { return callback(null, this.resolvers[codec]) diff --git a/src/util.js b/src/util.js new file mode 100644 index 0000000..1e86375 --- /dev/null +++ b/src/util.js @@ -0,0 +1,32 @@ +'use strict' + +exports.first = async (iterator) => { + for await (const value of iterator) { + return value + } +} + +exports.last = async (iterator) => { + let value + for await (value of iterator) { + // Intentionally empty + } + return value +} + +exports.all = async (iterator) => { + const values = [] + for await (const value of iterator) { + values.push(value) + } + return values +} + +exports.fancyIterator = (next) => { + const iterator = { next } + iterator[Symbol.asyncIterator] = function () { return this } + iterator.first = () => exports.first(iterator) + iterator.last = () => exports.last(iterator) + iterator.all = () => exports.all(iterator) + return iterator +} diff --git a/test/basics.js b/test/basics.js index 42ad2a8..1bcfea0 100644 --- a/test/basics.js +++ b/test/basics.js @@ -3,8 +3,10 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') +const chaiAsProised = require('chai-as-promised') const expect = chai.expect chai.use(dirtyChai) +chai.use(chaiAsProised) const BlockService = require('ipfs-block-service') const CID = require('cids') const multihash = require('multihashes') @@ -33,29 +35,28 @@ module.exports = (repo) => { }) describe('validation', () => { - it('get - errors on unknown resolver', (done) => { + it('resolve - errors on unknown resolver', async () => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) // choosing a format that is not supported const cid = new CID(1, 'base1', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1')) - r.get(cid, '/', {}, (err, result) => { - expect(err).to.exist() - expect(err.message).to.eql('No resolver found for codec "base1"') - done() - }) + const result = r.resolve(cid, '') + await expect(result.next()).to.be.rejectedWith( + 'No resolver found for codec "base1"') }) - it('_get - errors on unknown resolver', (done) => { - const bs = new BlockService(repo) - const r = new IPLDResolver({ blockService: bs }) - // choosing a format that is not supported - const cid = new CID(1, 'base1', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1')) - r.get(cid, (err, result) => { - expect(err).to.exist() - expect(err.message).to.eql('No resolver found for codec "base1"') - done() - }) - }) + // TODO vmx 2018-11-29 Change this test to use `get()`. + // it('_get - errors on unknown resolver', (done) => { + // const bs = new BlockService(repo) + // const r = new IPLDResolver({ blockService: bs }) + // // choosing a format that is not supported + // const cid = new CID(1, 'base1', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1')) + // r.get(cid, (err, result) => { + // expect(err).to.exist() + // expect(err.message).to.eql('No resolver found for codec "base1"') + // done() + // }) + // } it('put - errors on unknown resolver', (done) => { const bs = new BlockService(repo) diff --git a/test/format-support.js b/test/format-support.js index be05e1e..0cfb9da 100644 --- a/test/format-support.js +++ b/test/format-support.js @@ -3,8 +3,10 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') +const chaiAsProised = require('chai-as-promised') const expect = chai.expect chai.use(dirtyChai) +chai.use(chaiAsProised) const BlockService = require('ipfs-block-service') const dagCBOR = require('ipld-dag-cbor') @@ -28,21 +30,19 @@ module.exports = (repo) => { }) describe('Dynamic format loading', () => { - it('should fail to dynamically load format', (done) => { + it('should fail to dynamically load format', async () => { const bs = new BlockService(repo) const resolver = new IPLDResolver({ blockService: bs, formats: [] }) - resolver.get(cid, '/', (err) => { - expect(err).to.exist() - expect(err.message).to.equal('No resolver found for codec "dag-cbor"') - done() - }) + const result = resolver.resolve(cid, '') + await expect(result.next()).to.be.rejectedWith( + 'No resolver found for codec "dag-cbor"') }) - it('should fail to dynamically load format via loadFormat option', (done) => { + it('should fail to dynamically load format via loadFormat option', async () => { const errMsg = 'BOOM' + Date.now() const bs = new BlockService(repo) const resolver = new IPLDResolver({ @@ -54,14 +54,11 @@ module.exports = (repo) => { } }) - resolver.get(cid, '/', (err) => { - expect(err).to.exist() - expect(err.message).to.equal(errMsg) - done() - }) + const result = resolver.resolve(cid, '') + await expect(result.next()).to.be.rejectedWith(errMsg) }) - it('should dynamically load missing format', (done) => { + it('should dynamically load missing format', async () => { const bs = new BlockService(repo) const resolver = new IPLDResolver({ blockService: bs, @@ -72,14 +69,12 @@ module.exports = (repo) => { } }) - resolver.get(cid, '/', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(data) - done() - }) + const result = resolver.resolve(cid, '') + const node = await result.first() + expect(node.value).to.eql(data) }) - it('should not dynamically load format added statically', (done) => { + it('should not dynamically load format added statically', async () => { const bs = new BlockService(repo) const resolver = new IPLDResolver({ blockService: bs, @@ -89,11 +84,8 @@ module.exports = (repo) => { } }) - resolver.get(cid, '/', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(data) - done() - }) + const result = resolver.resolve(cid, '') + await result.next() }) }) }) diff --git a/test/ipld-all.js b/test/ipld-all.js index a75e50a..18b8443 100644 --- a/test/ipld-all.js +++ b/test/ipld-all.js @@ -61,12 +61,16 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { ], done) }) - it('resolve through different formats', (done) => { - resolver.get(cidCbor, 'pb/Data', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(Buffer.from('I am inside a Protobuf')) - done() - }) + it('resolve through different formats', async () => { + const result = resolver.resolve(cidCbor, 'pb/Data') + + const node1 = await result.first() + expect(node1.remainderPath).to.eql('Data') + expect(node1.value).to.eql(cidPb) + + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value).to.eql(Buffer.from('I am inside a Protobuf')) }) it('does not store nodes when onlyHash is passed', (done) => { diff --git a/test/ipld-bitcoin.js b/test/ipld-bitcoin.js index f3421f1..a97e7a7 100644 --- a/test/ipld-bitcoin.js +++ b/test/ipld-bitcoin.js @@ -107,16 +107,17 @@ module.exports = (repo) => { }, done) }) - it('resolver._get', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err, result) => { - expect(err).to.not.exist() - expect(node1.version).to.eql(result.value.version) - done() - }) - }) - }) + // TODO vmx 2018-11-30 Change this test to use `get()`. + // it('resolver._get', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err, result) => { + // expect(err).to.not.exist() + // expect(node1.version).to.eql(result.value.version) + // done() + // }) + // }) + // }) }) describe('public api', () => { @@ -150,62 +151,75 @@ module.exports = (repo) => { }) }) - it('root path (same as get)', (done) => { - resolver.get(cid1, '/', (err, result) => { - expect(err).to.not.exist() - - ipldBitcoin.util.cid(result.value, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.eql(cid1) - done() - }) - }) + // TODO vmx 2018-11-30: Implement getting the whole object properly + // it('root path (same as get)', (done) => { + // resolver.get(cid1, '/', (err, result) => { + // expect(err).to.not.exist() + // + // ipldBitcoin.util.cid(result.value, (err, cid) => { + // expect(err).to.not.exist() + // expect(cid).to.eql(cid1) + // done() + // }) + // }) + // }) + + it('resolves value within 1st node scope', async () => { + const result = resolver.resolve(cid1, 'version') + const node = await result.first() + expect(node.remainderPath).to.eql('') + expect(node.value).to.eql(1) }) - it('value within 1st node scope', (done) => { - resolver.get(cid1, 'version', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(1) - done() - }) - }) + it('resolves value within nested scope (1 level)', async () => { + const result = resolver.resolve(cid2, 'parent/version') - it('value within nested scope (1 level)', (done) => { - resolver.get(cid2, 'parent/version', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(1) - done() - }) - }) + const node1 = await result.first() + expect(node1.remainderPath).to.eql('version') + expect(node1.value).to.eql(cid1) - it('value within nested scope (2 levels)', (done) => { - resolver.get(cid3, 'parent/parent/version', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(1) - done() - }) + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value).to.eql(1) }) - it('resolver.remove', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err, result) => { - expect(err).to.not.exist() - expect(result.value.version).to.eql(1) - remove() - }) - }) + it('resolves value within nested scope (2 levels)', async () => { + const result = resolver.resolve(cid3, 'parent/parent/version') - function remove () { - resolver.remove(cid1, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err) => { - expect(err).to.exist() - done() - }) - }) - } + const node1 = await result.first() + expect(node1.remainderPath).to.eql('parent/version') + expect(node1.value).to.eql(cid2) + + const node2 = await result.first() + expect(node2.remainderPath).to.eql('version') + expect(node2.value).to.eql(cid1) + + const node3 = await result.first() + expect(node3.remainderPath).to.eql('') + expect(node3.value).to.eql(1) }) + + // // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` + // it('resolver.remove', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err, result) => { + // expect(err).to.not.exist() + // expect(result.value.version).to.eql(1) + // remove() + // }) + // }) + // + // function remove () { + // resolver.remove(cid1, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err) => { + // expect(err).to.exist() + // done() + // }) + // }) + // } + // }) }) }) } diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index eacc3b9..afba4d8 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -3,8 +3,10 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') +const chaiAsProised = require('chai-as-promised') const expect = chai.expect chai.use(dirtyChai) +chai.use(chaiAsProised) const BlockService = require('ipfs-block-service') const dagCBOR = require('ipld-dag-cbor') const series = require('async/series') @@ -91,16 +93,17 @@ module.exports = (repo) => { }, done) }) - it('resolver._get', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver._get(cid1, (err, node) => { - expect(err).to.not.exist() - expect(node1).to.eql(node) - done() - }) - }) - }) + // TODO vmx 2018-11-30 Change this test to use `get()`. + // it('resolver._get', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver._get(cid1, (err, node) => { + // expect(err).to.not.exist() + // expect(node1).to.eql(node) + // done() + // }) + // }) + // }) }) describe('public api', () => { @@ -134,129 +137,70 @@ module.exports = (repo) => { }) }) - it('resolver.get just CID', (done) => { - resolver.get(cid1, (err, result) => { - expect(err).to.not.exist() - - dagCBOR.util.cid(result.value, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.eql(cid1) - done() - }) - }) - }) - - it('resolver.get root path', (done) => { - resolver.get(cid1, '/', (err, result) => { - expect(err).to.not.exist() - - dagCBOR.util.cid(result.value, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.eql(cid1) - done() - }) - }) - }) - - it('resolver.get relative path `.` (same as get /)', (done) => { - resolver.get(cid1, '.', (err, result) => { - expect(err).to.not.exist() - - dagCBOR.util.cid(result.value, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.eql(cid1) - done() - }) - }) + // TODO vmx 2018-11-30: Implement getting the whole object properly + // it('resolver.get root path', (done) => { + // resolver.get(cid1, '/', (err, result) => { + // expect(err).to.not.exist() + // + // dagCBOR.util.cid(result.value, (err, cid) => { + // expect(err).to.not.exist() + // expect(cid).to.eql(cid1) + // done() + // }) + // }) + // }) + + it('resolves value within 1st node scope', async () => { + const result = resolver.resolve(cid1, 'someData') + const node = await result.first() + expect(node.remainderPath).to.eql('') + expect(node.value).to.eql('I am 1') }) - it('resolver.get relative path `./` (same as get /)', (done) => { - resolver.get(cid1, './', (err, result) => { - expect(err).to.not.exist() + it('resolves value within nested scope (0 level)', async () => { + const result = resolver.resolve(cid2, 'one') - dagCBOR.util.cid(result.value, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.eql(cid1) - done() - }) - }) - }) - - it('resolver.get relative path `./one/someData` (same as get one/someData)', (done) => { - resolver.get(cid2, './one/someData', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql('I am 1') - done() - }) - }) + const node1 = await result.first() + expect(node1.remainderPath).to.eql('') + expect(node1.value).to.eql(cid1) - it('resolver.get relative path `one/./someData` (same as get one/someData)', (done) => { - resolver.get(cid2, 'one/./someData', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql('I am 1') - done() - }) + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value).to.eql({ someData: 'I am 1' }) }) - it('resolver.get double slash at the beginning `//one/someData` (same as get one/someData)', (done) => { - resolver.get(cid2, '//one/someData', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql('I am 1') - done() - }) - }) + it('resolves value within nested scope (1 level)', async () => { + const result = resolver.resolve(cid2, 'one/someData') - it('resolver.get double slash in the middle `one//someData` (same as get one/someData)', (done) => { - resolver.get(cid2, 'one//someData', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql('I am 1') - done() - }) - }) + const node1 = await result.first() + expect(node1.remainderPath).to.eql('someData') + expect(node1.value).to.eql(cid1) - it('resolver.get value within 1st node scope', (done) => { - resolver.get(cid1, 'someData', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql('I am 1') - done() - }) + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value).to.eql('I am 1') }) - it('resolver.get value within nested scope (0 level)', (done) => { - resolver.get(cid2, 'one', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql({ - someData: 'I am 1' - }) - done() - }) - }) + it('resolves value within nested scope (2 levels)', async () => { + const result = resolver.resolve(cid3, 'two/one/someData') - it('resolver.get value within nested scope (1 level)', (done) => { - resolver.get(cid2, 'one/someData', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql('I am 1') - done() - }) - }) + const node1 = await result.first() + expect(node1.remainderPath).to.eql('one/someData') + expect(node1.value).to.eql(cid2) - it('resolver.get value within nested scope (2 levels)', (done) => { - resolver.get(cid3, 'two/one/someData', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql('I am 1') - expect(result.remainderPath).to.eql('') - expect(result.cid).to.deep.eql(cid1) + const node2 = await result.first() + expect(node2.remainderPath).to.eql('someData') + expect(node2.value).to.eql(cid1) - done() - }) + const node3 = await result.first() + expect(node3.remainderPath).to.eql('') + expect(node3.value).to.eql('I am 1') }) - it('resolver.get calls callback for unavailable path', (done) => { - resolver.get(cid3, `foo/${Date.now()}`, (err) => { - expect(err).to.exist() - expect(err.message).to.contain('path not available') - done() - }) + it('fails resolving unavailable path', async () => { + const result = resolver.resolve(cid3, `foo/${Date.now()}`) + await expect(result.next()).to.be.rejectedWith( + 'path not available at root') }) it('resolver.tree', (done) => { @@ -330,26 +274,27 @@ module.exports = (repo) => { ) }) - it('resolver.remove', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err, result) => { - expect(err).to.not.exist() - expect(node1).to.eql(result.value) - remove() - }) - }) - - function remove () { - resolver.remove(cid1, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err) => { - expect(err).to.exist() - done() - }) - }) - } - }) + // // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` + // it('resolver.remove', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err, result) => { + // expect(err).to.not.exist() + // expect(node1).to.eql(result.value) + // remove() + // }) + // }) + // + // function remove () { + // resolver.remove(cid1, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err) => { + // expect(err).to.exist() + // done() + // }) + // }) + // } + // }) }) }) } diff --git a/test/ipld-dag-pb.js b/test/ipld-dag-pb.js index bfadb8e..efffcfe 100644 --- a/test/ipld-dag-pb.js +++ b/test/ipld-dag-pb.js @@ -147,15 +147,16 @@ module.exports = (repo) => { }, done) }) - it('resolver._get', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver._get(cid1, (err, node) => { - expect(err).to.not.exist() - done() - }) - }) - }) + // TODO vmx 2018-11-29 Change this test to use `get()`. + // it('resolver._get', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver._get(cid1, (err, node) => { + // expect(err).to.not.exist() + // done() + // }) + // }) + // }) }) describe('public api', () => { @@ -189,98 +190,84 @@ module.exports = (repo) => { }) }) - it('resolver.get just CID', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (done)) - }) - }) + // TODO vmx 2018-11-29: Change this test to use the new `get()` + // it('resolver.get with empty path', (done) => { + // resolver.get(cid1, '/', (err, result) => { + // expect(err).to.not.exist() + // + // dagPB.util.cid(result.value, (err, cid) => { + // expect(err).to.not.exist() + // expect(cid).to.eql(cid1) + // done() + // }) + // }) + // }) - it('resolver.getStream', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - pull( - resolver.getStream(cid1), - pull.collect(done) - ) - }) + it('resolves a value within 1st node scope', async () => { + const result = resolver.resolve(cid1, 'Data') + const node = await result.first() + expect(node.remainderPath).to.eql('') + expect(node.value).to.eql(Buffer.from('I am 1')) }) - it('resolver.get root path', (done) => { - resolver.get(cid1, '/', (err, result) => { - expect(err).to.not.exist() + it('resolves a value within nested scope (1 level)', async () => { + const result = resolver.resolve(cid2, 'Links/0/Hash/Data') - dagPB.util.cid(result.value, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.eql(cid1) - done() - }) - }) - }) + const node1 = await result.first() + expect(node1.remainderPath).to.eql('Data') + expect(node1.value).to.eql(cid1) - it('resolver.get value within 1st node scope', (done) => { - resolver.get(cid1, 'Data', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(Buffer.from('I am 1')) - done() - }) + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value).to.eql(Buffer.from('I am 1')) }) - it('resolver.get value within nested scope (1 level)', (done) => { - resolver.get(cid2, 'Links/0/Hash/Data', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(Buffer.from('I am 1')) - done() - }) - }) + it('resolves value within nested scope (2 levels)', async () => { + const result = resolver.resolve(cid3, 'Links/1/Hash/Links/0/Hash/Data') - it('resolver.get value within nested scope (2 levels)', (done) => { - resolver.get(cid3, 'Links/1/Hash/Links/0/Hash/Data', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(Buffer.from('I am 1')) - done() - }) - }) + const node1 = await result.first() + expect(node1.remainderPath).to.eql('Links/0/Hash/Data') + expect(node1.value).to.eql(cid2) - it('resolver.get with option localResolve: true', (done) => { - resolver.get(cid3, 'Links/1/Hash/Links/0/Hash/Data', { localResolve: true }, (err, result) => { - expect(err).to.not.exist() - expect(result.remainderPath).to.equal('Links/0/Hash/Data') - expect(result.value).to.eql({ - '/': 'QmS149H7EbyMuZ2wtEF1sAd7gPwjj4rKAorweAjKMkxr8D' - }) - expect(result.cid).to.deep.equal(cid2) - done() - }) - }) + const node2 = await result.first() + expect(node2.remainderPath).to.eql('Data') + expect(node2.value).to.eql(cid1) - it('resolver.get value within nested scope (1 level) returns cid of node traversed to', (done) => { - resolver.get(cid2, 'Links/0/Hash/Data', (err, result) => { - expect(err).to.not.exist() - expect(result.cid).to.deep.equal(cid1) - done() - }) + const node3 = await result.first() + expect(node3.remainderPath).to.eql('') + expect(node3.value).to.eql(Buffer.from('I am 1')) }) - it('resolver.remove', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err, node) => { - expect(err).to.not.exist() - remove() - }) - }) + // TODO vmx 2018-11-29: Think about if every returned node should contain + // a `cid` field or not + // it('resolver.get value within nested scope (1 level) returns cid of node traversed to', (done) => { + // resolver.get(cid2, 'Links/0/Hash/Data', (err, result) => { + // expect(err).to.not.exist() + // expect(result.cid).to.deep.equal(cid1) + // done() + // }) + // }) - function remove () { - resolver.remove(cid1, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err) => { - expect(err).to.exist() - done() - }) - }) - } - }) + // TODO vmx 2018-11-29: remove this `get()` call with the new `get()` + // it('resolver.remove', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err, node) => { + // expect(err).to.not.exist() + // remove() + // }) + // }) + // + // function remove () { + // resolver.remove(cid1, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err) => { + // expect(err).to.exist() + // done() + // }) + // }) + // } + // }) }) }) } diff --git a/test/ipld-eth-block.js b/test/ipld-eth-block.js index bf86d16..1231380 100644 --- a/test/ipld-eth-block.js +++ b/test/ipld-eth-block.js @@ -95,18 +95,19 @@ module.exports = (repo) => { }, done) }) - it('resolver._get', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err, result) => { - expect(err).to.not.exist() - expect(node1.number.toString('hex')).to.eql('01') - expect(node1.raw).to.eql(result.value.raw) - expect(node1.hash()).to.eql(result.value.hash()) - done() - }) - }) - }) + // TODO vmx 2018-11-30 Change this test to use `get()`. + // it('resolver._get', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err, result) => { + // expect(err).to.not.exist() + // expect(node1.number.toString('hex')).to.eql('01') + // expect(node1.raw).to.eql(result.value.raw) + // expect(node1.hash()).to.eql(result.value.hash()) + // done() + // }) + // }) + // }) }) describe('public api', () => { @@ -140,64 +141,77 @@ module.exports = (repo) => { }) }) - it('root path (same as get)', (done) => { - resolver.get(cid1, '/', (err, result) => { - expect(err).to.not.exist() - - ipldEthBlock.util.cid(result.value, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.eql(cid1) - done() - }) - }) + // TODO vmx 2018-11-30: Implement getting the whole object properly + // it('root path (same as get)', (done) => { + // resolver.get(cid1, '/', (err, result) => { + // expect(err).to.not.exist() + // + // ipldEthBlock.util.cid(result.value, (err, cid) => { + // expect(err).to.not.exist() + // expect(cid).to.eql(cid1) + // done() + // }) + // }) + // }) + + it('resolves value within 1st node scope', async () => { + const result = resolver.resolve(cid1, 'number') + const node = await result.first() + expect(node.remainderPath).to.eql('') + expect(node.value.toString('hex')).to.eql('01') }) - it('value within 1st node scope', (done) => { - resolver.get(cid1, 'number', (err, result) => { - expect(err).to.not.exist() - expect(result.value.toString('hex')).to.eql('01') - done() - }) - }) + it('resolves value within nested scope (1 level)', async () => { + const result = resolver.resolve(cid2, 'parent/number') - it('value within nested scope (1 level)', (done) => { - resolver.get(cid2, 'parent/number', (err, result) => { - expect(err).to.not.exist() - expect(result.value.toString('hex')).to.eql('01') - done() - }) - }) + const node1 = await result.first() + expect(node1.remainderPath).to.eql('number') + expect(node1.value).to.eql(cid1) - it('value within nested scope (2 levels)', (done) => { - resolver.get(cid3, 'parent/parent/number', (err, result) => { - expect(err).to.not.exist() - expect(result.value.toString('hex')).to.eql('01') - done() - }) + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value.toString('hex')).to.eql('01') }) - it('resolver.remove', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err, result) => { - expect(err).to.not.exist() - const node = result.value - expect(node1.raw).to.eql(node.raw) - expect(node1.hash()).to.eql(node.hash()) - remove() - }) - }) + it('resolves value within nested scope (2 levels)', async () => { + const result = resolver.resolve(cid3, 'parent/parent/number') - function remove () { - resolver.remove(cid1, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err) => { - expect(err).to.exist() - done() - }) - }) - } + const node1 = await result.first() + expect(node1.remainderPath).to.eql('parent/number') + expect(node1.value).to.eql(cid2) + + const node2 = await result.first() + expect(node2.remainderPath).to.eql('number') + expect(node2.value).to.eql(cid1) + + const node3 = await result.first() + expect(node3.remainderPath).to.eql('') + expect(node3.value.toString('hex')).to.eql('01') }) + + // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` + // it('resolver.remove', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err, result) => { + // expect(err).to.not.exist() + // const node = result.value + // expect(node1.raw).to.eql(node.raw) + // expect(node1.hash()).to.eql(node.hash()) + // remove() + // }) + // }) + // + // function remove () { + // resolver.remove(cid1, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err) => { + // expect(err).to.exist() + // done() + // }) + // }) + // } + // }) }) }) } diff --git a/test/ipld-eth.js b/test/ipld-eth.js index 9a02004..0e1579e 100644 --- a/test/ipld-eth.js +++ b/test/ipld-eth.js @@ -93,23 +93,24 @@ module.exports = (repo) => { } }) - describe('resolver.get', () => { - it('block-to-block', (done) => { - resolver.get(ethObjs.child.cid, '/parent', (err, result) => { - expect(err).to.not.exist() - expect(result.remainderPath).to.equal('') - expect(result.value.number.toString('hex')).to.equal('302516') - done() - }) + describe('resolver.resolve', () => { + it('block-to-block', async () => { + const result = resolver.resolve(ethObjs.child.cid, 'parent') + + const node1 = await result.first() + expect(node1.remainderPath).to.eql('') + + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value.number.toString('hex')).to.eql('302516') }) - it('block-to-account resolve', (done) => { - resolver.get(ethObjs.child.cid, '/parent/state/0/0/0/0/1/7/2/7/8/a/1/e/6/e/9/6/3/5/e/1/a/3/f/1/1/e/b/0/2/2/d/a/1/f/5/7/e/a/0/0/4/d/8/5/2/d/9/d/1/9/4/2/d/4/3/6/0/8/5/4/0/4/7/1/nonce', (err, result) => { - expect(err).to.not.exist() - expect(result.value.toString('hex'), '03') - expect(result.remainderPath).to.equal('') - done() - }) + it('block-to-account resolve', async () => { + const result = resolver.resolve(ethObjs.child.cid, + 'parent/state/0/0/0/0/1/7/2/7/8/a/1/e/6/e/9/6/3/5/e/1/a/3/f/1/1/e/b/0/2/2/d/a/1/f/5/7/e/a/0/0/4/d/8/5/2/d/9/d/1/9/4/2/d/4/3/6/0/8/5/4/0/4/7/1/nonce') + const node = await result.last() + expect(node.value.toString('hex'), '03') + expect(node.remainderPath).to.equal('') }) }) }) diff --git a/test/ipld-git.js b/test/ipld-git.js index 059b0f3..529986f 100644 --- a/test/ipld-git.js +++ b/test/ipld-git.js @@ -162,16 +162,17 @@ module.exports = (repo) => { }, done) }) - it('resolver._get', (done) => { - resolver.put(blobNode, { cid: blobCid }, (err) => { - expect(err).to.not.exist() - resolver.get(blobCid, (err, result) => { - expect(err).to.not.exist() - expect(blobNode.toString('hex')).to.eql(result.value.toString('hex')) - done() - }) - }) - }) + // TODO vmx 2018-11-30 Change this test to use `get()`. + // it('resolver._get', (done) => { + // resolver.put(blobNode, { cid: blobCid }, (err) => { + // expect(err).to.not.exist() + // resolver.get(blobCid, (err, result) => { + // expect(err).to.not.exist() + // expect(blobNode.toString('hex')).to.eql(result.value.toString('hex')) + // done() + // }) + // }) + // }) }) describe('public api', () => { @@ -205,79 +206,103 @@ module.exports = (repo) => { }) }) - it('resolver.get root path', (done) => { - resolver.get(blobCid, '/', (err, result) => { - expect(err).to.not.exist() + // TODO vmx 2018-11-30: Implement getting the whole object properly + // it('resolver.get empty path', (done) => { + // resolver.get(blobCid, '', (err, result) => { + // expect(err).to.not.exist() + // + // ipldGit.util.cid(result.value, (err, cid) => { + // expect(err).to.not.exist() + // expect(cid).to.eql(blobCid) + // done() + // }) + // }) + // }) - ipldGit.util.cid(result.value, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.eql(blobCid) - done() - }) - }) + it('resolves value within 1st node scope', async () => { + const result = resolver.resolve(commitCid, 'message') + const node = await result.first() + expect(node.remainderPath).to.eql('') + expect(node.value).to.eql('Initial commit\n') }) - it('value within 1st node scope', (done) => { - resolver.get(commitCid, 'message', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql('Initial commit\n') - done() - }) - }) + it('resolves value within nested node scope (commit/tree)', async () => { + const result = resolver.resolve(commitCid, 'tree/somefile/mode') - it('value within nested node scope (commit/tree)', (done) => { - resolver.get(commitCid, 'tree/somefile/mode', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql('100644') - done() - }) - }) + const node1 = await result.first() + expect(node1.remainderPath).to.eql('somefile/mode') + expect(node1.value).to.eql(treeCid) - it('value within nested node scope (commit/tree/blob)', (done) => { - resolver.get(commitCid, 'tree/somefile/hash', (err, result) => { - expect(err).to.not.exist() - expect(blobNode.toString('hex')).to.eql(result.value.toString('hex')) - done() - }) + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value).to.eql('100644') }) - it('value within nested node scope (commit/commit/tree/blob)', (done) => { - resolver.get(commit2Cid, 'parents/0/tree/somefile/hash', (err, result) => { - expect(err).to.not.exist() - expect(blobNode.toString('hex')).to.eql(result.value.toString('hex')) - done() - }) + it('resolves value within nested node scope (commit/tree/blob)', async () => { + const result = resolver.resolve(commitCid, 'tree/somefile/hash') + + const node1 = await result.first() + expect(node1.remainderPath).to.eql('somefile/hash') + expect(node1.value).to.eql(treeCid) + + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value).to.eql(blobCid) + + const node3 = await result.first() + expect(node3.remainderPath).to.eql('') + expect(node3.value).to.eql(blobNode) }) - it('value within nested node scope (tag/commit/commit/tree/blob)', (done) => { - resolver.get(tagCid, 'object/parents/0/tree/somefile/hash', (err, result) => { - expect(err).to.not.exist() - expect(blobNode.toString('hex')).to.eql(result.value.toString('hex')) - done() - }) + it('resolves value within nested node scope (commit/commit/tree/blob)', async () => { + const result = resolver.resolve(commit2Cid, 'parents/0/tree/somefile/hash') + + const node1 = await result.first() + expect(node1.remainderPath).to.eql('tree/somefile/hash') + expect(node1.value).to.eql(commitCid) + + // The nodes in between were already tested by some other test + const last = await result.last() + expect(last.remainderPath).to.eql('') + expect(last.value).to.eql(blobNode) }) - it('resolver.remove', (done) => { - resolver.put(blobNode, { cid: blobCid }, (err) => { - expect(err).to.not.exist() - resolver.get(blobCid, (err, result) => { - expect(err).to.not.exist() - const node = result.value - expect(blobNode.toString('hex')).to.eql(node.toString('hex')) - remove() - }) - }) + it('resolves value within nested node scope (tag/commit/commit/tree/blob)', async () => { + const result = resolver.resolve(tagCid, + 'object/parents/0/tree/somefile/hash') - function remove () { - resolver.remove(blobCid, (err) => { - expect(err).to.not.exist() - resolver.get(blobCid, (err) => { - expect(err).to.exist() - done() - }) - }) - } + const node1 = await result.first() + expect(node1.remainderPath).to.eql('parents/0/tree/somefile/hash') + expect(node1.value).to.eql(commit2Cid) + + // The nodes in between were already tested by some other test + const last = await result.last() + expect(last.remainderPath).to.eql('') + expect(last.value).to.eql(blobNode) }) + + // // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` + // it('resolver.remove', (done) => { + // resolver.put(blobNode, { cid: blobCid }, (err) => { + // expect(err).to.not.exist() + // resolver.get(blobCid, (err, result) => { + // expect(err).to.not.exist() + // const node = result.value + // expect(blobNode.toString('hex')).to.eql(node.toString('hex')) + // remove() + // }) + // }) + // + // function remove () { + // resolver.remove(blobCid, (err) => { + // expect(err).to.not.exist() + // resolver.get(blobCid, (err) => { + // expect(err).to.exist() + // done() + // }) + // }) + // } + // }) }) }) } diff --git a/test/ipld-zcash.js b/test/ipld-zcash.js index c8bba30..0980e5d 100644 --- a/test/ipld-zcash.js +++ b/test/ipld-zcash.js @@ -112,16 +112,17 @@ module.exports = (repo) => { }, done) }) - it('resolver._get', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err, result) => { - expect(err).to.not.exist() - expect(node1.version).to.eql(result.value.version) - done() - }) - }) - }) + // // TODO vmx 2018-11-30 Change this test to use `get()`. + // it('resolver._get', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err, result) => { + // expect(err).to.not.exist() + // expect(node1.version).to.eql(result.value.version) + // done() + // }) + // }) + // }) }) describe('public api', () => { @@ -155,62 +156,75 @@ module.exports = (repo) => { }) }) - it('root path (same as get)', (done) => { - resolver.get(cid1, '/', (err, result) => { - expect(err).to.not.exist() - - ipldZcash.util.cid(result.value, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.eql(cid1) - done() - }) - }) + // // TODO vmx 2018-11-30: Implement getting the whole object properly + // it('root path (same as get)', (done) => { + // resolver.get(cid1, '/', (err, result) => { + // expect(err).to.not.exist() + // + // ipldZcash.util.cid(result.value, (err, cid) => { + // expect(err).to.not.exist() + // expect(cid).to.eql(cid1) + // done() + // }) + // }) + // }) + + it('resolves value within 1st node scope', async () => { + const result = resolver.resolve(cid1, 'version') + const node = await result.first() + expect(node.remainderPath).to.eql('') + expect(node.value).to.eql(1) }) - it('value within 1st node scope', (done) => { - resolver.get(cid1, 'version', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(1) - done() - }) - }) + it('resolves value within nested scope (1 level)', async () => { + const result = resolver.resolve(cid2, 'parent/version') - it('value within nested scope (1 level)', (done) => { - resolver.get(cid2, 'parent/version', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(1) - done() - }) - }) + const node1 = await result.first() + expect(node1.remainderPath).to.eql('version') + expect(node1.value).to.eql(cid1) - it('value within nested scope (2 levels)', (done) => { - resolver.get(cid3, 'parent/parent/version', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.eql(1) - done() - }) + const node2 = await result.first() + expect(node2.remainderPath).to.eql('') + expect(node2.value).to.eql(1) }) - it('resolver.remove', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err, result) => { - expect(err).to.not.exist() - expect(result.value.version).to.eql(1) - remove() - }) - }) + it('resolves value within nested scope (2 levels)', async () => { + const result = resolver.resolve(cid3, 'parent/parent/version') - function remove () { - resolver.remove(cid1, (err) => { - expect(err).to.not.exist() - resolver.get(cid1, (err) => { - expect(err).to.exist() - done() - }) - }) - } + const node1 = await result.first() + expect(node1.remainderPath).to.eql('parent/version') + expect(node1.value).to.eql(cid2) + + const node2 = await result.first() + expect(node2.remainderPath).to.eql('version') + expect(node2.value).to.eql(cid1) + + const node3 = await result.first() + expect(node3.remainderPath).to.eql('') + expect(node3.value).to.eql(1) }) + + // // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` + // it('resolver.remove', (done) => { + // resolver.put(node1, { cid: cid1 }, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err, result) => { + // expect(err).to.not.exist() + // expect(result.value.version).to.eql(1) + // remove() + // }) + // }) + // + // function remove () { + // resolver.remove(cid1, (err) => { + // expect(err).to.not.exist() + // resolver.get(cid1, (err) => { + // expect(err).to.exist() + // done() + // }) + // }) + // } + // }) }) }) } From cb2551d7b522eee2118233bb49cbc42c5d624ba3 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Fri, 7 Dec 2018 00:45:06 +0100 Subject: [PATCH 02/27] refactor: make `_getFormat()` async/await BREAKING CHANGE: your custom format loading function needs to be an async now. So the signature for `options.loadFormat` is no longer: function (codec, callback) but async functiont (codec) --- src/index.js | 113 ++++++++++++++++++++++++----------------- test/format-support.js | 12 ++--- 2 files changed, 71 insertions(+), 54 deletions(-) diff --git a/src/index.js b/src/index.js index c0fc840..494938b 100644 --- a/src/index.js +++ b/src/index.js @@ -42,9 +42,13 @@ class IPLDResolver { } } - this.support.load = options.loadFormat || ((codec, callback) => { - callback(new Error(`No resolver found for codec "${codec}"`)) - }) + if (options.loadFormat === undefined) { + this.support.load = async (codec) => { + throw new Error(`No resolver found for codec "${codec}"`) + } + } else { + this.support.load = options.loadFormat + } this.support.rm = (multicodec) => { if (this.resolvers[multicodec]) { @@ -83,46 +87,47 @@ class IPLDResolver { return Promise.resolve({ done: true }) } - return new Promise((resolve, reject) => { - this._getFormat(cid.codec, (err, format) => { + return new Promise(async (resolve, reject) => { + let format + try { + format = await this._getFormat(cid.codec) + } catch (err) { + return reject(err) + } + + // get block + // use local resolver + // update path value + this.bs.get(cid, (err, block) => { if (err) { return reject(err) } - // get block - // use local resolver - // update path value - this.bs.get(cid, (err, block) => { + format.resolver.resolve(block.data, path, (err, result) => { if (err) { return reject(err) } - format.resolver.resolve(block.data, path, (err, result) => { - if (err) { - return reject(err) - } + // Prepare for the next iteration if there is a `remainderPath` + path = result.remainderPath + let value = result.value + // NOTE vmx 2018-11-29: Not all IPLD Formats return links as + // CIDs yet. Hence try to convert old style links to CIDs + if (Object.keys(value).length === 1 && '/' in value) { + value = new CID(value['/']) + } + if (CID.isCID(value)) { + cid = value + } else { + cid = null + } - // Prepare for the next iteration if there is a `remainderPath` - path = result.remainderPath - let value = result.value - // NOTE vmx 2018-11-29: Not all IPLD Formats return links as - // CIDs yet. Hence try to convert old style links to CIDs - if (Object.keys(value).length === 1 && '/' in value) { - value = new CID(value['/']) - } - if (CID.isCID(value)) { - cid = value - } else { - cid = null + return resolve({ + done: false, + value: { + remainderPath: path, + value } - - return resolve({ - done: false, - value: { - remainderPath: path, - value - } - }) }) }) }) @@ -148,9 +153,12 @@ class IPLDResolver { return callback(err) } map(blocks, (block, mapCallback) => { - this._getFormat(block.cid.codec, (err, format) => { - if (err) return mapCallback(err) + // TODO vmx 2018-12-07: Make this one async/await once + // `util.serialize()` is a Promise + this._getFormat(block.cid.codec).then((format) => { format.util.deserialize(block.data, mapCallback) + }).catch((err) => { + mapCallback(err) }) }, callback) @@ -174,9 +182,9 @@ class IPLDResolver { return this._put(options.cid, node, callback) } - this._getFormat(options.format, (err, format) => { - if (err) return callback(err) - + // TODO vmx 2018-12-07: Make this async/await once `put()` returns a + // Promise + this._getFormat(options.format).then((format) => { format.util.cid(node, options, (err, cid) => { if (err) { return callback(err) @@ -188,6 +196,8 @@ class IPLDResolver { this._put(cid, node, callback) }) + }).catch((err) => { + callback(err) }) } @@ -205,7 +215,9 @@ class IPLDResolver { p = pullDeferSource() waterfall([ - (cb) => this._getFormat(cid.codec, cb), + async () => { + return this._getFormat(cid.codec) + }, (format, cb) => this.bs.get(cid, (err, block) => { if (err) return cb(err) cb(null, format, block) @@ -239,7 +251,9 @@ class IPLDResolver { const cid = el.cid waterfall([ - (cb) => this._getFormat(cid.codec, cb), + async () => { + return this._getFormat(cid.codec) + }, (format, cb) => this.bs.get(cid, (err, block) => { if (err) return cb(err) cb(null, format, block) @@ -310,24 +324,27 @@ class IPLDResolver { /* */ /* internals */ /* */ - _getFormat (codec, callback) { + async _getFormat (codec) { if (this.resolvers[codec]) { - return callback(null, this.resolvers[codec]) + return this.resolvers[codec] } // If not supported, attempt to dynamically load this format - this.support.load(codec, (err, format) => { - if (err) return callback(err) - this.resolvers[codec] = format - callback(null, format) - }) + const format = await this.support.load(codec) + this.resolvers[codec] = format + return format } _put (cid, node, callback) { callback = callback || noop waterfall([ - (cb) => this._getFormat(cid.codec, cb), + (cb) => { + this._getFormat(cid.codec).then( + (format) => cb(null, format), + (error) => cb(error) + ) + }, (format, cb) => format.util.serialize(node, cb), (buf, cb) => this.bs.put(new Block(buf, cid), cb) ], (err) => { diff --git a/test/format-support.js b/test/format-support.js index 0cfb9da..54ab6a5 100644 --- a/test/format-support.js +++ b/test/format-support.js @@ -48,9 +48,9 @@ module.exports = (repo) => { const resolver = new IPLDResolver({ blockService: bs, formats: [], - loadFormat (codec, callback) { - if (codec !== 'dag-cbor') return callback(new Error('unexpected codec')) - setTimeout(() => callback(new Error(errMsg))) + async loadFormat (codec) { + if (codec !== 'dag-cbor') throw new Error('unexpected codec') + throw new Error(errMsg) } }) @@ -63,9 +63,9 @@ module.exports = (repo) => { const resolver = new IPLDResolver({ blockService: bs, formats: [], - loadFormat (codec, callback) { - if (codec !== 'dag-cbor') return callback(new Error('unexpected codec')) - setTimeout(() => callback(null, dagCBOR)) + async loadFormat (codec) { + if (codec !== 'dag-cbor') throw new Error('unexpected codec') + return dagCBOR } }) From 2026f9f49db5d781e22ec62c41cb8c742f5b37c2 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Fri, 25 Jan 2019 18:21:58 +0100 Subject: [PATCH 03/27] refactor: store codecs by their code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of storing a codec by their name, use their code instead. The tests are changed from `base1` to `blake2b-8` as codecs for different bases are no longer part of multicodec, but are part of multibase now. BREAKING CHANGE: The `codec` parameter in `options.loadFormat()` is a number Instead of returnign the name of the codec as string, the codec code (a number) is now returned. So if you e.g. check within the function for a certain format, it changes from: async loadFormat (codec) { if (codec !== 'dag-cbor') … } To: async loadFormat (codec) { if (codec !== multicodec.DAG_CBOR) … } --- package.json | 1 + src/index.js | 32 ++++++++++++++++++++++---------- test/basics.js | 28 ++++++++++++++++++++-------- test/format-support.js | 9 +++++++-- 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index fd5f7ba..348c20f 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "ipld-dag-pb": "~0.15.2", "ipld-raw": "^2.0.1", "merge-options": "^1.0.1", + "multicodec": "~0.5.0", "pull-defer": "~0.2.3", "pull-stream": "^3.6.9", "pull-traverse": "^1.0.3" diff --git a/src/index.js b/src/index.js index 494938b..3b14b70 100644 --- a/src/index.js +++ b/src/index.js @@ -11,6 +11,7 @@ const mergeOptions = require('merge-options') const ipldDagCbor = require('ipld-dag-cbor') const ipldDagPb = require('ipld-dag-pb') const ipldRaw = require('ipld-raw') +const multicodec = require('multicodec') const { fancyIterator } = require('./util') function noop () {} @@ -31,12 +32,13 @@ class IPLDResolver { this.support = {} // Adds support for an IPLD format - this.support.add = (multicodec, resolver, util) => { - if (this.resolvers[multicodec]) { - throw new Error('Resolver already exists for codec "' + multicodec + '"') + this.support.add = (codec, resolver, util) => { + if (this.resolvers[codec]) { + const codecName = multicodec.print[codec] + throw new Error(`Resolver already exists for codec "${codecName}"`) } - this.resolvers[multicodec] = { + this.resolvers[codec] = { resolver: resolver, util: util } @@ -44,23 +46,26 @@ class IPLDResolver { if (options.loadFormat === undefined) { this.support.load = async (codec) => { - throw new Error(`No resolver found for codec "${codec}"`) + const codecName = multicodec.print[codec] + throw new Error(`No resolver found for codec "${codecName}"`) } } else { this.support.load = options.loadFormat } - this.support.rm = (multicodec) => { - if (this.resolvers[multicodec]) { - delete this.resolvers[multicodec] + this.support.rm = (codec) => { + if (this.resolvers[codec]) { + delete this.resolvers[codec] } } // Enable all supplied formats for (const format of options.formats) { const { resolver, util } = format - const multicodec = resolver.multicodec - this.support.add(multicodec, resolver, util) + // IPLD Formats are using strings instead of constants for the multicodec + const codecBuffer = multicodec.getCodeVarint(resolver.multicodec) + const codec = multicodec.getCode(codecBuffer) + this.support.add(codec, resolver, util) } } @@ -325,6 +330,13 @@ class IPLDResolver { /* internals */ /* */ async _getFormat (codec) { + // TODO vmx 2019-01-24: Once all CIDs support accessing the codec code + // instead of the name, remove this part + if (typeof codec === 'string') { + const constantName = codec.toUpperCase().replace(/-/g, '_') + codec = multicodec[constantName] + } + if (this.resolvers[codec]) { return this.resolvers[codec] } diff --git a/test/basics.js b/test/basics.js index 1bcfea0..1ac432d 100644 --- a/test/basics.js +++ b/test/basics.js @@ -39,10 +39,14 @@ module.exports = (repo) => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) // choosing a format that is not supported - const cid = new CID(1, 'base1', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1')) + const cid = new CID( + 1, + 'blake2b-8', + multihash.encode(Buffer.from('abcd', 'hex'), 'sha1') + ) const result = r.resolve(cid, '') await expect(result.next()).to.be.rejectedWith( - 'No resolver found for codec "base1"') + 'No resolver found for codec "blake2b-8"') }) // TODO vmx 2018-11-29 Change this test to use `get()`. @@ -62,9 +66,9 @@ module.exports = (repo) => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) // choosing a format that is not supported - r.put(null, { format: 'base1' }, (err, result) => { + r.put(null, { format: 'blake2b-8' }, (err, result) => { expect(err).to.exist() - expect(err.message).to.eql('No resolver found for codec "base1"') + expect(err.message).to.eql('No resolver found for codec "blake2b-8"') done() }) }) @@ -83,10 +87,14 @@ module.exports = (repo) => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) // choosing a format that is not supported - const cid = new CID(1, 'base1', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1')) + const cid = new CID( + 1, + 'blake2b-8', + multihash.encode(Buffer.from('abcd', 'hex'), 'sha1') + ) r._put(cid, null, (err, result) => { expect(err).to.exist() - expect(err.message).to.eql('No resolver found for codec "base1"') + expect(err.message).to.eql('No resolver found for codec "blake2b-8"') done() }) }) @@ -95,12 +103,16 @@ module.exports = (repo) => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) // choosing a format that is not supported - const cid = new CID(1, 'base1', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1')) + const cid = new CID( + 1, + 'blake2b-8', + multihash.encode(Buffer.from('abcd', 'hex'), 'sha1') + ) pull( r.treeStream(cid, '/', {}), pull.collect(function (err) { expect(err).to.exist() - expect(err.message).to.eql('No resolver found for codec "base1"') + expect(err.message).to.eql('No resolver found for codec "blake2b-8"') done() }) ) diff --git a/test/format-support.js b/test/format-support.js index 54ab6a5..60785ec 100644 --- a/test/format-support.js +++ b/test/format-support.js @@ -9,6 +9,7 @@ chai.use(dirtyChai) chai.use(chaiAsProised) const BlockService = require('ipfs-block-service') const dagCBOR = require('ipld-dag-cbor') +const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -49,7 +50,9 @@ module.exports = (repo) => { blockService: bs, formats: [], async loadFormat (codec) { - if (codec !== 'dag-cbor') throw new Error('unexpected codec') + if (codec !== multicodec.DAG_CBOR) { + throw new Error('unexpected codec') + } throw new Error(errMsg) } }) @@ -64,7 +67,9 @@ module.exports = (repo) => { blockService: bs, formats: [], async loadFormat (codec) { - if (codec !== 'dag-cbor') throw new Error('unexpected codec') + if (codec !== multicodec.DAG_CBOR) { + throw new Error('unexpected codec') + } return dagCBOR } }) From c9dab0f0364e98ec20d57f5366eaa925f3bedae8 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Fri, 7 Dec 2018 22:40:08 +0100 Subject: [PATCH 04/27] feat: implementation of the new `put()` function BREAKING CHANGE: The API of `put()` changes. The API docs for it: > Stores the given IPLD Nodes of a recognized IPLD Format. - `nodes` (`Iterable`): deserialized IPLD nodes that should be inserted. - `format` (`multicodec`, required): the multicodec of the format that IPLD Node should be encoded in. - `options` is applied to any of the `nodes` and is an object with the following properties: - `hashAlg` (`multicodec`, default: hash algorithm of the given multicodec): the hashing algorithm that is used to calculate the CID. - `cidVersion` (`boolean`, default: 1): the CID version to use. - `onlyHash` (`boolean`, default: false): if true the serialized form of the IPLD Node will not be passed to the underlying block store. Returns an async iterator with the CIDs of the serialized IPLD Nodes. --- package.json | 3 +- src/index.js | 93 ++++++++++++++++++++++++---------- test/basics.js | 19 +++---- test/format-support.js | 9 ++-- test/ipld-all.js | 53 ++++++++------------ test/ipld-bitcoin.js | 60 +++++++++------------- test/ipld-dag-cbor.js | 60 ++++++++++------------ test/ipld-dag-pb.js | 92 ++++++++++------------------------ test/ipld-eth-block.js | 111 +++++++++++++---------------------------- test/ipld-eth.js | 47 ++++++++--------- test/ipld-git.js | 62 +++++++++-------------- test/ipld-zcash.js | 60 +++++++++------------- 12 files changed, 279 insertions(+), 390 deletions(-) diff --git a/package.json b/package.json index 348c20f..7c7306f 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,8 @@ "multicodec": "~0.5.0", "pull-defer": "~0.2.3", "pull-stream": "^3.6.9", - "pull-traverse": "^1.0.3" + "pull-traverse": "^1.0.3", + "typical": "^3.0.0" }, "contributors": [ "Alan Shaw ", diff --git a/src/index.js b/src/index.js index 3b14b70..690c015 100644 --- a/src/index.js +++ b/src/index.js @@ -12,6 +12,7 @@ const ipldDagCbor = require('ipld-dag-cbor') const ipldDagPb = require('ipld-dag-pb') const ipldRaw = require('ipld-raw') const multicodec = require('multicodec') +const typical = require('typical') const { fancyIterator } = require('./util') function noop () {} @@ -170,40 +171,82 @@ class IPLDResolver { }) } - put (node, options, callback) { - if (typeof options === 'function') { - callback = options - return setImmediate(() => callback( - new Error('IPLDResolver.put requires options') - )) + /** + * Stores the given IPLD Nodes of a recognized IPLD Format. + * + * @param {Iterable.} nodes - Deserialized IPLD nodes that should be inserted. + * @param {number} format - The multicodec of the format that IPLD Node should be encoded in. + * @param {Object} [userOptions] - Options are applied to any of the `nodes` and is an object with the following properties. + * @param {number} [userOtions.hashAlg=hash algorithm of the given multicodec] - The hashing algorithm that is used to calculate the CID. + * @param {number} [userOptions.cidVersion=1]`- The CID version to use. + * @param {boolean} [userOptions.onlyHash=false] - If true the serialized form of the IPLD Node will not be passed to the underlying block store. + * @returns {Iterable.>} - Returns an async iterator with the CIDs of the serialized IPLD Nodes. + */ + put (nodes, format, userOptions) { + if (!typical.isIterable(nodes) || typical.isString(nodes) || + Buffer.isBuffer(nodes)) { + throw new Error('`nodes` must be an iterable') + } + if (format === undefined) { + throw new Error('`put` requires a format') + } + if (typeof format !== 'number') { + throw new Error('`format` parameter must be number (multicodec)') } - callback = callback || noop - if (options.cid && CID.isCID(options.cid)) { - if (options.onlyHash) { - return setImmediate(() => callback(null, options.cid)) - } + let options + let formatImpl - return this._put(options.cid, node, callback) - } + const next = () => { + // End iteration if there are no more nodes to put + if (nodes.length === 0) { + return Promise.resolve({ done: true }) + } - // TODO vmx 2018-12-07: Make this async/await once `put()` returns a - // Promise - this._getFormat(options.format).then((format) => { - format.util.cid(node, options, (err, cid) => { - if (err) { - return callback(err) + return new Promise(async (resolve, reject) => { + // Lazy load the options not when the iterator is initialized, but + // when we hit the first iteration. This way the constructor can be + // a synchronous function. + if (options === undefined) { + try { + formatImpl = await this._getFormat(format) + } catch (err) { + return reject(err) + } + const defaultOptions = { + hashAlg: formatImpl.defaultHashAlg, + cidVersion: 1, + onlyHash: false + } + options = mergeOptions(defaultOptions, userOptions) } - if (options.onlyHash) { - return callback(null, cid) + const node = nodes.shift() + const cidOptions = { + version: options.cidVersion, + hashAlg: options.hashAlg, + onlyHash: options.onlyHash } + formatImpl.util.cid(node, cidOptions, (err, cid) => { + if (err) { + return reject(err) + } - this._put(cid, node, callback) + if (options.onlyHash) { + return resolve({ done: false, value: cid }) + } + + this._put(cid, node, (err, cid) => { + if (err) { + return reject(err) + } + return resolve({ done: false, value: cid }) + }) + }) }) - }).catch((err) => { - callback(err) - }) + } + + return fancyIterator(next) } treeStream (cid, path, options) { diff --git a/test/basics.js b/test/basics.js index 1ac432d..a88fc46 100644 --- a/test/basics.js +++ b/test/basics.js @@ -10,6 +10,7 @@ chai.use(chaiAsProised) const BlockService = require('ipfs-block-service') const CID = require('cids') const multihash = require('multihashes') +const multicodec = require('multicodec') const pull = require('pull-stream') const inMemory = require('ipld-in-memory') @@ -62,25 +63,19 @@ module.exports = (repo) => { // }) // } - it('put - errors on unknown resolver', (done) => { + it('put - errors on unknown resolver', async () => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) // choosing a format that is not supported - r.put(null, { format: 'blake2b-8' }, (err, result) => { - expect(err).to.exist() - expect(err.message).to.eql('No resolver found for codec "blake2b-8"') - done() - }) + const result = r.put([null], multicodec.BLAKE2B_8) + await expect(result.next()).to.be.rejectedWith( + 'No resolver found for codec "blake2b-8"') }) - it('put - errors if no options', (done) => { + it('put - errors if no format is provided', () => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) - r.put(null, (err, result) => { - expect(err).to.exist() - expect(err.message).to.eql('IPLDResolver.put requires options') - done() - }) + expect(() => r.put([null])).to.be.throw('`put` requires a format') }) it('_put - errors on unknown resolver', (done) => { diff --git a/test/format-support.js b/test/format-support.js index 60785ec..dc362d3 100644 --- a/test/format-support.js +++ b/test/format-support.js @@ -17,17 +17,14 @@ module.exports = (repo) => { describe('IPLD format support', () => { let data, cid - before((done) => { + before(async () => { const bs = new BlockService(repo) const resolver = new IPLDResolver({ blockService: bs }) data = { now: Date.now() } - dagCBOR.util.cid(data, (err, c) => { - expect(err).to.not.exist() - cid = c - resolver.put(data, { cid }, done) - }) + const result = resolver.put([data], multicodec.DAG_CBOR) + cid = await result.last() }) describe('Dynamic format loading', () => { diff --git a/test/ipld-all.js b/test/ipld-all.js index 18b8443..7b5bd0b 100644 --- a/test/ipld-all.js +++ b/test/ipld-all.js @@ -17,6 +17,7 @@ const each = require('async/each') const waterfall = require('async/waterfall') const CID = require('cids') const inMemory = require('ipld-in-memory') +const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -52,10 +53,15 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { cidCbor = cid each([ - { node: nodePb, cid: cidPb }, - { node: nodeCbor, cid: cidCbor } + { node: nodePb, format: multicodec.DAG_PB, cidVersion: 0 }, + { node: nodeCbor, format: multicodec.DAG_CBOR, cidVersion: 1 } ], (nac, cb) => { - resolver.put(nac.node, { cid: nac.cid }, cb) + resolver.put([nac.node], nac.format, { + cidVersion: nac.cidVersion + }).first().then( + () => cb(null), + (error) => cb(error) + ) }, cb) } ], done) @@ -76,12 +82,17 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { it('does not store nodes when onlyHash is passed', (done) => { waterfall([ (cb) => dagPB.DAGNode.create(Buffer.from('Some data here'), cb), - (node, cb) => resolver.put(node, { - onlyHash: true, - version: 1, - hashAlg: 'sha2-256', - format: 'dag-pb' - }, cb), + (node, cb) => { + const result = resolver.put([node], multicodec.DAG_PB, { + onlyHash: true, + cidVersion: 1, + hashAlg: multicodec.SHA2_256 + }) + result.first().then( + (cid) => cb(null, cid), + (error) => cb(error) + ) + }, (cid, cb) => resolver.bs._repo.blocks.has(cid, cb) ], (error, result) => { if (error) { @@ -93,30 +104,6 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { }) }) - it('does not store nodes when onlyHash is passed and a CID is passed', (done) => { - const cid = new CID('QmTmxQfEHbQzntsXPTU4ae2ZgBGwseBmS12AkZnKCkuf2G') - - waterfall([ - (cb) => dagPB.DAGNode.create(Buffer.from('Some data here'), cb), - (node, cb) => resolver.put(node, { - onlyHash: true, - cid - }, cb), - (cid2, cb) => { - expect(cid2).to.equal(cid) - - resolver.bs._repo.blocks.has(cid2, cb) - } - ], (error, result) => { - if (error) { - return done(error) - } - - expect(result).to.be.false() - done() - }) - }) - describe('getMany', () => { it('should return nodes correctly', (done) => { resolver.getMany([cidCbor, cidPb], (err, result) => { diff --git a/test/ipld-bitcoin.js b/test/ipld-bitcoin.js index a97e7a7..bf7f6a9 100644 --- a/test/ipld-bitcoin.js +++ b/test/ipld-bitcoin.js @@ -11,7 +11,7 @@ const BitcoinBlock = require('bitcoinjs-lib').Block const multihash = require('multihashes') const series = require('async/series') const each = require('async/each') -const pull = require('pull-stream') +const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -83,16 +83,12 @@ module.exports = (repo) => { } ], store) - function store () { - pull( - pull.values([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ]), - pull.asyncMap((nac, cb) => resolver.put(nac.node, { cid: nac.cid }, cb)), - pull.onEnd(done) - ) + async function store () { + const nodes = [node1, node2, node3] + const result = resolver.put(nodes, multicodec.BITCOIN_BLOCK) + ;[cid1, cid2, cid3] = await result.all() + + done() } }) @@ -121,34 +117,26 @@ module.exports = (repo) => { }) describe('public api', () => { - it('resolver.put', (done) => { - resolver.put(node1, { cid: cid1 }, done) - }) - - it('resolver.put with format', (done) => { - resolver.put(node1, { format: 'bitcoin-block' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('bitcoin-block') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('dbl-sha2-256') - done() - }) + it('resolver.put with format', async () => { + const result = resolver.put([node1], multicodec.BITCOIN_BLOCK) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('bitcoin-block') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('dbl-sha2-256') }) - it('resolver.put with format + hashAlg', (done) => { - resolver.put(node1, { format: 'bitcoin-block', hashAlg: 'sha3-512' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('bitcoin-block') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('sha3-512') - done() + it('resolver.put with format + hashAlg', async () => { + const result = resolver.put([node1], multicodec.BITCOIN_BLOCK, { + hashAlg: multicodec.SHA3_512 }) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('bitcoin-block') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('sha3-512') }) // TODO vmx 2018-11-30: Implement getting the whole object properly diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index afba4d8..4d99743 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -12,6 +12,7 @@ const dagCBOR = require('ipld-dag-cbor') const series = require('async/series') const each = require('async/each') const pull = require('pull-stream') +const multicodec = require('multicodec') const multihash = require('multihashes') const IPLDResolver = require('../src') @@ -69,16 +70,12 @@ module.exports = (repo) => { } ], store) - function store () { - pull( - pull.values([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ]), - pull.asyncMap((nac, cb) => resolver.put(nac.node, { cid: nac.cid }, cb)), - pull.onEnd(done) - ) + async function store () { + const nodes = [node1, node2, node3] + const result = resolver.put(nodes, multicodec.DAG_CBOR) + ;[cid1, cid2, cid3] = await result.all() + + done() } }) @@ -107,34 +104,27 @@ module.exports = (repo) => { }) describe('public api', () => { - it('resolver.put with CID', (done) => { - resolver.put(node1, { cid: cid1 }, done) - }) - - it('resolver.put with format', (done) => { - resolver.put(node1, { format: 'dag-cbor' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('dag-cbor') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('sha2-256') - done() - }) + it('resolver.put with format', async () => { + const result = resolver.put([node1], multicodec.DAG_CBOR) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('dag-cbor') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('sha2-256') }) - it('resolver.put with format + hashAlg', (done) => { - resolver.put(node1, { format: 'dag-cbor', hashAlg: 'sha3-512' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('dag-cbor') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('sha3-512') - done() + it('resolver.put with format + hashAlg', async () => { + const result = resolver.put([node1], multicodec.DAG_CBOR, { + hashAlg: multicodec.SHA3_512 }) + const cid = await result.first() + expect(cid).to.exist() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('dag-cbor') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('sha3-512') }) // TODO vmx 2018-11-30: Implement getting the whole object properly diff --git a/test/ipld-dag-pb.js b/test/ipld-dag-pb.js index efffcfe..bb2eeaa 100644 --- a/test/ipld-dag-pb.js +++ b/test/ipld-dag-pb.js @@ -9,8 +9,8 @@ const BlockService = require('ipfs-block-service') const dagPB = require('ipld-dag-pb') const series = require('async/series') const each = require('async/each') -const pull = require('pull-stream') const multihash = require('multihashes') +const multicodec = require('multicodec') const IPLDResolver = require('../src') module.exports = (repo) => { @@ -95,44 +95,14 @@ module.exports = (repo) => { }) }) } - ], cids) + ], store) - function cids () { - series([ - (cb) => { - dagPB.util.cid(node1, (err, cid) => { - expect(err).to.not.exist() - cid1 = cid - cb() - }) - }, - (cb) => { - dagPB.util.cid(node2, (err, cid) => { - expect(err).to.not.exist() - cid2 = cid - cb() - }) - }, - (cb) => { - dagPB.util.cid(node3, (err, cid) => { - expect(err).to.not.exist() - cid3 = cid - cb() - }) - } - ], store) - } + async function store () { + const nodes = [node1, node2, node3] + const result = resolver.put(nodes, multicodec.DAG_PB) + ;[cid1, cid2, cid3] = await result.all() - function store () { - pull( - pull.values([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ]), - pull.asyncMap((nac, cb) => resolver.put(nac.node, { cid: nac.cid }, cb)), - pull.onEnd(done) - ) + done() } }) @@ -160,34 +130,26 @@ module.exports = (repo) => { }) describe('public api', () => { - it('resolver.put with CID', (done) => { - resolver.put(node1, { cid: cid1 }, done) - }) - - it('resolver.put with format', (done) => { - resolver.put(node1, { format: 'dag-pb' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(0) - expect(cid.codec).to.equal('dag-pb') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('sha2-256') - done() - }) + it('resolver.put with format', async () => { + const result = resolver.put([node1], multicodec.DAG_PB) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('dag-pb') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('sha2-256') }) - it('resolver.put with format + hashAlg', (done) => { - resolver.put(node1, { format: 'dag-pb', hashAlg: 'sha3-512' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('dag-pb') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('sha3-512') - done() + it('resolver.put with format + hashAlg', async () => { + const result = resolver.put([node1], multicodec.DAG_PB, { + hashAlg: multicodec.SHA3_512 }) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('dag-pb') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('sha3-512') }) // TODO vmx 2018-11-29: Change this test to use the new `get()` @@ -215,7 +177,7 @@ module.exports = (repo) => { const node1 = await result.first() expect(node1.remainderPath).to.eql('Data') - expect(node1.value).to.eql(cid1) + expect(node1.value).to.eql(cid1.toV0()) const node2 = await result.first() expect(node2.remainderPath).to.eql('') @@ -227,11 +189,11 @@ module.exports = (repo) => { const node1 = await result.first() expect(node1.remainderPath).to.eql('Links/0/Hash/Data') - expect(node1.value).to.eql(cid2) + expect(node1.value).to.eql(cid2.toV0()) const node2 = await result.first() expect(node2.remainderPath).to.eql('Data') - expect(node2.value).to.eql(cid1) + expect(node2.value).to.eql(cid1.toV0()) const node3 = await result.first() expect(node3.remainderPath).to.eql('') diff --git a/test/ipld-eth-block.js b/test/ipld-eth-block.js index 1231380..72f25fa 100644 --- a/test/ipld-eth-block.js +++ b/test/ipld-eth-block.js @@ -9,9 +9,8 @@ const BlockService = require('ipfs-block-service') const ipldEthBlock = require('ipld-ethereum').ethBlock const EthBlockHeader = require('ethereumjs-block/header') const multihash = require('multihashes') -const series = require('async/series') const each = require('async/each') -const pull = require('pull-stream') +const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -26,62 +25,28 @@ module.exports = (repo) => { let cid2 let cid3 - before((done) => { + before(async () => { const bs = new BlockService(repo) resolver = new IPLDResolver({ blockService: bs, formats: [ipldEthBlock] }) - series([ - (cb) => { - node1 = new EthBlockHeader({ - number: 1 - }) - - ipldEthBlock.util.cid(node1, (err, cid) => { - expect(err).to.not.exist() - cid1 = cid - cb() - }) - }, - (cb) => { - node2 = new EthBlockHeader({ - number: 2, - parentHash: node1.hash() - }) - - ipldEthBlock.util.cid(node2, (err, cid) => { - expect(err).to.not.exist() - cid2 = cid - cb() - }) - }, - (cb) => { - node3 = new EthBlockHeader({ - number: 3, - parentHash: node2.hash() - }) - - ipldEthBlock.util.cid(node3, (err, cid) => { - expect(err).to.not.exist() - cid3 = cid - cb() - }) - } - ], store) - - function store () { - pull( - pull.values([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ]), - pull.asyncMap((nac, cb) => resolver.put(nac.node, { cid: nac.cid }, cb)), - pull.onEnd(done) - ) - } + node1 = new EthBlockHeader({ + number: 1 + }) + node2 = new EthBlockHeader({ + number: 2, + parentHash: node1.hash() + }) + node3 = new EthBlockHeader({ + number: 3, + parentHash: node2.hash() + }) + + const nodes = [node1, node2, node3] + const result = resolver.put(nodes, multicodec.ETH_BLOCK) + ;[cid1, cid2, cid3] = await result.all() }) describe('internals', () => { @@ -111,34 +76,26 @@ module.exports = (repo) => { }) describe('public api', () => { - it('resolver.put', (done) => { - resolver.put(node1, { cid: cid1 }, done) - }) - - it('resolver.put with format', (done) => { - resolver.put(node1, { format: 'eth-block' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('eth-block') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('keccak-256') - done() - }) + it('resolver.put with format', async () => { + const result = resolver.put([node1], multicodec.ETH_BLOCK) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('eth-block') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('keccak-256') }) - it('resolver.put with format + hashAlg', (done) => { - resolver.put(node1, { format: 'eth-block', hashAlg: 'keccak-512' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('eth-block') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('keccak-512') - done() + it('resolver.put with format + hashAlg', async () => { + const result = resolver.put([node1], multicodec.ETH_BLOCK, { + hashAlg: multicodec.KECCAK_512 }) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('eth-block') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('keccak-512') }) // TODO vmx 2018-11-30: Implement getting the whole object properly diff --git a/test/ipld-eth.js b/test/ipld-eth.js index 0e1579e..ed6a200 100644 --- a/test/ipld-eth.js +++ b/test/ipld-eth.js @@ -13,8 +13,7 @@ const loadFixture = require('aegir/fixtures') const async = require('async') const EthBlockHeader = require('ethereumjs-block/header') const EthTrieNode = require('merkle-patricia-tree/trieNode') -const multihashes = require('multihashes') -const CID = require('cids') +const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -34,8 +33,7 @@ module.exports = (repo) => { async.waterfall([ readFilesFixture, - generateCids, - putInStore + generateCids ], done) function readFilesFixture (cb) { @@ -54,48 +52,44 @@ module.exports = (repo) => { function generateCids (fileData, cb) { ethObjs = { - child: generateForType('child', 'eth-block', fileData.child), - block: generateForType('block', 'eth-block', fileData.block), - stateRoot: generateForType('stateRoot', 'eth-state-trie', fileData.stateRoot), - state0: generateForType('state0', 'eth-state-trie', fileData.state0), - state00: generateForType('state00', 'eth-state-trie', fileData.state00), - state000: generateForType('state000', 'eth-state-trie', fileData.state000), - state0000: generateForType('state0000', 'eth-state-trie', fileData.state0000), - state00001: generateForType('state00001', 'eth-state-trie', fileData.state00001), - state000017: generateForType('state000017', 'eth-state-trie', fileData.state000017) + child: generateForType('child', multicodec.ETH_BLOCK, fileData.child), + block: generateForType('block', multicodec.ETH_BLOCK, fileData.block), + stateRoot: generateForType('stateRoot', multicodec.ETH_STATE_TRIE, fileData.stateRoot), + state0: generateForType('state0', multicodec.ETH_STATE_TRIE, fileData.state0), + state00: generateForType('state00', multicodec.ETH_STATE_TRIE, fileData.state00), + state000: generateForType('state000', multicodec.ETH_STATE_TRIE, fileData.state000), + state0000: generateForType('state0000', multicodec.ETH_STATE_TRIE, fileData.state0000), + state00001: generateForType('state00001', multicodec.ETH_STATE_TRIE, fileData.state00001), + state000017: generateForType('state000017', multicodec.ETH_STATE_TRIE, fileData.state000017) } cb() } - function generateForType (label, type, rawData) { + async function generateForType (label, type, rawData) { let node switch (type) { - case 'eth-block': node = new EthBlockHeader(rawData); break - case 'eth-state-trie': node = new EthTrieNode(rlp.decode(rawData)); break + case multicodec.ETH_BLOCK: node = new EthBlockHeader(rawData); break + case multicodec.ETH_STATE_TRIE: node = new EthTrieNode(rlp.decode(rawData)); break default: throw new Error('Unknown type!') } - const multihash = multihashes.encode(node.hash(), 'keccak-256') - const cid = new CID(1, type, multihash) + const result = resolver.put([node], type) + const cid = await result.first() + return { raw: rawData, node: node, cid: cid } } - - function putInStore (cb) { - async.each(ethObjs, (nodeData, next) => { - resolver.put(nodeData.node, { cid: nodeData.cid }, next) - }, cb) - } }) describe('resolver.resolve', () => { it('block-to-block', async () => { - const result = resolver.resolve(ethObjs.child.cid, 'parent') + const child = await ethObjs.child + const result = resolver.resolve(child.cid, 'parent') const node1 = await result.first() expect(node1.remainderPath).to.eql('') @@ -106,7 +100,8 @@ module.exports = (repo) => { }) it('block-to-account resolve', async () => { - const result = resolver.resolve(ethObjs.child.cid, + const child = await ethObjs.child + const result = resolver.resolve(child.cid, 'parent/state/0/0/0/0/1/7/2/7/8/a/1/e/6/e/9/6/3/5/e/1/a/3/f/1/1/e/b/0/2/2/d/a/1/f/5/7/e/a/0/0/4/d/8/5/2/d/9/d/1/9/4/2/d/4/3/6/0/8/5/4/0/4/7/1/nonce') const node = await result.last() expect(node.value.toString('hex'), '03') diff --git a/test/ipld-git.js b/test/ipld-git.js index 529986f..1111432 100644 --- a/test/ipld-git.js +++ b/test/ipld-git.js @@ -10,7 +10,7 @@ const ipldGit = require('ipld-git') const multihash = require('multihashes') const series = require('async/series') const each = require('async/each') -const pull = require('pull-stream') +const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -134,18 +134,12 @@ module.exports = (repo) => { } ], store) - function store () { - pull( - pull.values([ - { node: blobNode, cid: blobCid }, - { node: treeNode, cid: treeCid }, - { node: commitNode, cid: commitCid }, - { node: commit2Node, cid: commit2Cid }, - { node: tagNode, cid: tagCid } - ]), - pull.asyncMap((nac, cb) => resolver.put(nac.node, { cid: nac.cid }, cb)), - pull.onEnd(done) - ) + async function store () { + const nodes = [blobNode, treeNode, commitNode, commit2Node, tagNode] + const result = resolver.put(nodes, multicodec.GIT_RAW) + ;[blobCid, treeCid, commitCid, commit2Cid, tagCid] = await result.all() + + done() } }) @@ -176,34 +170,26 @@ module.exports = (repo) => { }) describe('public api', () => { - it('resolver.put', (done) => { - resolver.put(blobNode, { cid: blobCid }, done) - }) - - it('resolver.put with format', (done) => { - resolver.put(blobNode, { format: 'git-raw' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('git-raw') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('sha1') - done() - }) + it('resolver.put with format', async () => { + const result = resolver.put([blobNode], multicodec.GIT_RAW) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('git-raw') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('sha1') }) - it('resolver.put with format + hashAlg', (done) => { - resolver.put(blobNode, { format: 'git-raw', hashAlg: 'sha3-512' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('git-raw') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('sha3-512') - done() + it('resolver.put with format + hashAlg', async () => { + const result = resolver.put([blobNode], multicodec.GIT_RAW, { + hashAlg: multicodec.SHA3_512 }) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('git-raw') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('sha3-512') }) // TODO vmx 2018-11-30: Implement getting the whole object properly diff --git a/test/ipld-zcash.js b/test/ipld-zcash.js index 0980e5d..19b4c39 100644 --- a/test/ipld-zcash.js +++ b/test/ipld-zcash.js @@ -11,7 +11,7 @@ const ZcashBlockHeader = require('zcash-bitcore-lib').BlockHeader const multihash = require('multihashes') const series = require('async/series') const each = require('async/each') -const pull = require('pull-stream') +const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -88,16 +88,12 @@ module.exports = (repo) => { } ], store) - function store () { - pull( - pull.values([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ]), - pull.asyncMap((nac, cb) => resolver.put(nac.node, { cid: nac.cid }, cb)), - pull.onEnd(done) - ) + async function store () { + const nodes = [node1, node2, node3] + const result = resolver.put(nodes, multicodec.ZCASH_BLOCK) + ;[cid1, cid2, cid3] = await result.all() + + done() } }) @@ -126,34 +122,26 @@ module.exports = (repo) => { }) describe('public api', () => { - it('resolver.put', (done) => { - resolver.put(node1, { cid: cid1 }, done) - }) - - it('resolver.put with format', (done) => { - resolver.put(node1, { format: 'zcash-block' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('zcash-block') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('dbl-sha2-256') - done() - }) + it('resolver.put with format', async () => { + const result = resolver.put([node1], multicodec.ZCASH_BLOCK) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('zcash-block') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('dbl-sha2-256') }) - it('resolver.put with format + hashAlg', (done) => { - resolver.put(node1, { format: 'zcash-block', hashAlg: 'sha3-512' }, (err, cid) => { - expect(err).to.not.exist() - expect(cid).to.exist() - expect(cid.version).to.equal(1) - expect(cid.codec).to.equal('zcash-block') - expect(cid.multihash).to.exist() - const mh = multihash.decode(cid.multihash) - expect(mh.name).to.equal('sha3-512') - done() + it('resolver.put with format + hashAlg', async () => { + const result = resolver.put([node1], multicodec.ZCASH_BLOCK, { + hashAlg: multicodec.SHA3_512 }) + const cid = await result.first() + expect(cid.version).to.equal(1) + expect(cid.codec).to.equal('zcash-block') + expect(cid.multihash).to.exist() + const mh = multihash.decode(cid.multihash) + expect(mh.name).to.equal('sha3-512') }) // // TODO vmx 2018-11-30: Implement getting the whole object properly From 1f233126929a2a430b7903aec521eca8e515f5fe Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 12 Dec 2018 14:55:45 +0100 Subject: [PATCH 05/27] feat: implementation of the new `get()` function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: `get()` is replacing the `getMany()` function. The API docs for it: > Retrieve several IPLD Nodes at once. - `cids` (`Iterable`): the CIDs of the IPLD Nodes that should be retrieved. Returns an async iterator with the IPLD Nodes that correspond to the given `cids`. Throws an error if a IPLD Node can’t be retrieved. --- src/index.js | 90 ++++++++++++++++++++++++++++-------- test/basics.js | 13 ------ test/ipld-all.js | 69 +++++++++++++++------------- test/ipld-bitcoin.js | 74 +++++++++++------------------- test/ipld-dag-cbor.js | 33 ++++--------- test/ipld-dag-pb.js | 102 +++++++++++++++++++---------------------- test/ipld-eth-block.js | 81 +++++++++++++------------------- test/ipld-git.js | 75 +++++++++++------------------- test/ipld-zcash.js | 74 +++++++++++------------------- 9 files changed, 277 insertions(+), 334 deletions(-) diff --git a/src/index.js b/src/index.js index 690c015..9dfa7d6 100644 --- a/src/index.js +++ b/src/index.js @@ -146,29 +146,56 @@ class IPLDResolver { /** * Get multiple nodes back from an array of CIDs. * - * @param {Array} cids - * @param {function(Error, Array)} callback - * @returns {void} + * @param {Iterable.} cids - The CIDs of the IPLD Nodes that should be retrieved. + * @returns {Iterable.>} - Returns an async iterator with the IPLD Nodes that correspond to the given `cids`. */ - getMany (cids, callback) { - if (!Array.isArray(cids)) { - return callback(new Error('Argument must be an array of CIDs')) + get (cids) { + if (!typical.isIterable(cids) || typical.isString(cids) || + Buffer.isBuffer(cids)) { + throw new Error('`cids` must be an iterable of CIDs') } - this.bs.getMany(cids, (err, blocks) => { - if (err) { - return callback(err) + + let blocks + const next = () => { + // End of iteration if there aren't any blocks left to return + if (cids.length === 0 || + (blocks !== undefined && blocks.length === 0) + ) { + return Promise.resolve({ done: true }) } - map(blocks, (block, mapCallback) => { - // TODO vmx 2018-12-07: Make this one async/await once - // `util.serialize()` is a Promise - this._getFormat(block.cid.codec).then((format) => { - format.util.deserialize(block.data, mapCallback) - }).catch((err) => { - mapCallback(err) - }) - }, - callback) - }) + + return new Promise(async (resolve, reject) => { + // Lazy load block. + // Currntly the BlockService return all nodes as an array. In the + // future this will also be an iterator + if (blocks === undefined) { + const cidsArray = Array.from(cids) + this.bs.getMany(cidsArray, async (err, returnedBlocks) => { + if (err) { + return reject(err) + } + blocks = returnedBlocks + const block = blocks.shift() + try { + const node = await this._deserialize(block) + return resolve({ done: false, value: node }) + } catch (err) { + return reject(err) + } + }) + } else { + const block = blocks.shift() + try { + const node = await this._deserialize(block) + return resolve({ done: false, value: node }) + } catch (err) { + return reject(err) + } + } + }) + } + + return fancyIterator(next) } /** @@ -410,6 +437,29 @@ class IPLDResolver { }) } + /** + * Deserialize a given block + * + * @param {Object} block - The block to deserialize + * @return {Object} = Returns the deserialized node + */ + async _deserialize (block) { + return new Promise((resolve, reject) => { + this._getFormat(block.cid.codec).then((format) => { + // TODO vmx 2018-12-11: Make this one async/await once + // `util.serialize()` is a Promise + format.util.deserialize(block.data, (err, deserialized) => { + if (err) { + return reject(err) + } + return resolve(deserialized) + }) + }).catch((err) => { + return reject(err) + }) + }) + } + /** * Return a CID instance if it is a link. * diff --git a/test/basics.js b/test/basics.js index a88fc46..a6d0eaa 100644 --- a/test/basics.js +++ b/test/basics.js @@ -50,19 +50,6 @@ module.exports = (repo) => { 'No resolver found for codec "blake2b-8"') }) - // TODO vmx 2018-11-29 Change this test to use `get()`. - // it('_get - errors on unknown resolver', (done) => { - // const bs = new BlockService(repo) - // const r = new IPLDResolver({ blockService: bs }) - // // choosing a format that is not supported - // const cid = new CID(1, 'base1', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1')) - // r.get(cid, (err, result) => { - // expect(err).to.exist() - // expect(err.message).to.eql('No resolver found for codec "base1"') - // done() - // }) - // } - it('put - errors on unknown resolver', async () => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) diff --git a/test/ipld-all.js b/test/ipld-all.js index 7b5bd0b..d21a5f0 100644 --- a/test/ipld-all.js +++ b/test/ipld-all.js @@ -8,8 +8,10 @@ */ const chai = require('chai') +const chaiAsProised = require('chai-as-promised') const dirtyChai = require('dirty-chai') const expect = chai.expect +chai.use(chaiAsProised) chai.use(dirtyChai) const dagPB = require('ipld-dag-pb') const dagCBOR = require('ipld-dag-cbor') @@ -104,49 +106,50 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { }) }) - describe('getMany', () => { - it('should return nodes correctly', (done) => { - resolver.getMany([cidCbor, cidPb], (err, result) => { - expect(err).to.not.exist() - expect(result.length).to.equal(2) - expect(result).to.deep.equal([nodeCbor, nodePb]) - done() - }) + describe('get', () => { + it('should return nodes correctly', async () => { + const result = resolver.get([cidCbor, cidPb]) + const node1 = await result.first() + expect(node1).to.eql(nodeCbor) + + const node2 = await result.first() + expect(node2).to.eql(nodePb) }) - it('should return nodes in input order', (done) => { - resolver.getMany([cidPb, cidCbor], (err, result) => { - expect(err).to.not.exist() - expect(result.length).to.equal(2) - expect(result).to.deep.equal([nodePb, nodeCbor]) - done() - }) + it('should return nodes in input order', async () => { + const result = resolver.get([cidPb, cidCbor]) + const node1 = await result.first() + expect(node1).to.eql(nodePb) + + const node2 = await result.first() + expect(node2).to.eql(nodeCbor) }) - it('should return error on invalid CID', (done) => { - resolver.getMany([cidCbor, 'invalidcid'], (err, result) => { - expect(err.message).to.equal('Not a valid cid') - expect(result).to.be.undefined() - done() - }) + it('should return error on invalid CID', async () => { + const result = resolver.get([cidCbor, 'invalidcid']) + // TODO vmx 2018-12-11: This should really fail on the second node + // we get, as the first one is valid. This is only possible once + // the `getmany()` call of the BlockService takes and returns an + // iterator and not an array. + await expect(result.next()).to.be.rejectedWith( + 'Not a valid cid') }) - it('should return error on non-existent CID', (done) => { + it('should return error on non-existent CID', async () => { const nonExistentCid = new CID( 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - resolver.getMany([cidCbor, nonExistentCid], (err, result) => { - expect(err.message).to.equal('Not Found') - expect(result).to.be.undefined() - done() - }) + const result = resolver.get([cidCbor, nonExistentCid]) + // TODO vmx 2018-12-11: This should really fail on the second node + // we get, as the first one is valid. This is only possible once + // the `getmany()` call of the BlockService takes and returns an + // iterator and not an array. + await expect(result.next()).to.be.rejectedWith( + 'Not Found') }) - it('should return error on invalid input', (done) => { - resolver.getMany('astring', (err, result) => { - expect(err.message).to.equal('Argument must be an array of CIDs') - expect(result).to.be.undefined() - done() - }) + it('should return error on invalid input', () => { + expect(() => resolver.get('astring')).to.throw( + '`cids` must be an iterable of CIDs') }) }) }) diff --git a/test/ipld-bitcoin.js b/test/ipld-bitcoin.js index bf7f6a9..c3c6a75 100644 --- a/test/ipld-bitcoin.js +++ b/test/ipld-bitcoin.js @@ -102,18 +102,6 @@ module.exports = (repo) => { resolver._put(nc.cid, nc.node, cb) }, done) }) - - // TODO vmx 2018-11-30 Change this test to use `get()`. - // it('resolver._get', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err, result) => { - // expect(err).to.not.exist() - // expect(node1.version).to.eql(result.value.version) - // done() - // }) - // }) - // }) }) describe('public api', () => { @@ -139,19 +127,6 @@ module.exports = (repo) => { expect(mh.name).to.equal('sha3-512') }) - // TODO vmx 2018-11-30: Implement getting the whole object properly - // it('root path (same as get)', (done) => { - // resolver.get(cid1, '/', (err, result) => { - // expect(err).to.not.exist() - // - // ipldBitcoin.util.cid(result.value, (err, cid) => { - // expect(err).to.not.exist() - // expect(cid).to.eql(cid1) - // done() - // }) - // }) - // }) - it('resolves value within 1st node scope', async () => { const result = resolver.resolve(cid1, 'version') const node = await result.first() @@ -187,27 +162,34 @@ module.exports = (repo) => { expect(node3.value).to.eql(1) }) - // // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` - // it('resolver.remove', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err, result) => { - // expect(err).to.not.exist() - // expect(result.value.version).to.eql(1) - // remove() - // }) - // }) - // - // function remove () { - // resolver.remove(cid1, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err) => { - // expect(err).to.exist() - // done() - // }) - // }) - // } - // }) + it('resolver.get round-trip', async () => { + const resultPut = resolver.put([node1], multicodec.BITCOIN_BLOCK) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const node = await resultGet.first() + expect(node).to.deep.equal(node1) + }) + + it('resolver.remove', async () => { + const resultPut = resolver.put([node1], multicodec.BITCOIN_BLOCK) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const sameAsNode1 = await resultGet.first() + expect(sameAsNode1).to.deep.equal(node1) + return remove() + + function remove () { + return new Promise((resolve, reject) => { + resolver.remove(cid, (err) => { + expect(err).to.not.exist() + const resultGet = resolver.get([cid]) + expect(resultGet.next()).to.eventually.be.rejected() + .then(() => resolve()) + .catch((err) => reject(err)) + }) + }) + } + }) }) }) } diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index 4d99743..5a5d4e1 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -89,18 +89,6 @@ module.exports = (repo) => { resolver._put(nc.cid, nc.node, cb) }, done) }) - - // TODO vmx 2018-11-30 Change this test to use `get()`. - // it('resolver._get', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver._get(cid1, (err, node) => { - // expect(err).to.not.exist() - // expect(node1).to.eql(node) - // done() - // }) - // }) - // }) }) describe('public api', () => { @@ -127,19 +115,6 @@ module.exports = (repo) => { expect(mh.name).to.equal('sha3-512') }) - // TODO vmx 2018-11-30: Implement getting the whole object properly - // it('resolver.get root path', (done) => { - // resolver.get(cid1, '/', (err, result) => { - // expect(err).to.not.exist() - // - // dagCBOR.util.cid(result.value, (err, cid) => { - // expect(err).to.not.exist() - // expect(cid).to.eql(cid1) - // done() - // }) - // }) - // }) - it('resolves value within 1st node scope', async () => { const result = resolver.resolve(cid1, 'someData') const node = await result.first() @@ -193,6 +168,14 @@ module.exports = (repo) => { 'path not available at root') }) + it('resolver.get round-trip', async () => { + const resultPut = resolver.put([node1], multicodec.DAG_CBOR) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const node = await resultGet.first() + expect(node).to.deep.equal(node1) + }) + it('resolver.tree', (done) => { pull( resolver.treeStream(cid3), diff --git a/test/ipld-dag-pb.js b/test/ipld-dag-pb.js index bb2eeaa..6908256 100644 --- a/test/ipld-dag-pb.js +++ b/test/ipld-dag-pb.js @@ -2,8 +2,10 @@ 'use strict' const chai = require('chai') +const chaiAsProised = require('chai-as-promised') const dirtyChai = require('dirty-chai') const expect = chai.expect +chai.use(chaiAsProised) chai.use(dirtyChai) const BlockService = require('ipfs-block-service') const dagPB = require('ipld-dag-pb') @@ -116,17 +118,6 @@ module.exports = (repo) => { resolver._put(nc.cid, nc.node, cb) }, done) }) - - // TODO vmx 2018-11-29 Change this test to use `get()`. - // it('resolver._get', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver._get(cid1, (err, node) => { - // expect(err).to.not.exist() - // done() - // }) - // }) - // }) }) describe('public api', () => { @@ -152,19 +143,6 @@ module.exports = (repo) => { expect(mh.name).to.equal('sha3-512') }) - // TODO vmx 2018-11-29: Change this test to use the new `get()` - // it('resolver.get with empty path', (done) => { - // resolver.get(cid1, '/', (err, result) => { - // expect(err).to.not.exist() - // - // dagPB.util.cid(result.value, (err, cid) => { - // expect(err).to.not.exist() - // expect(cid).to.eql(cid1) - // done() - // }) - // }) - // }) - it('resolves a value within 1st node scope', async () => { const result = resolver.resolve(cid1, 'Data') const node = await result.first() @@ -200,36 +178,52 @@ module.exports = (repo) => { expect(node3.value).to.eql(Buffer.from('I am 1')) }) - // TODO vmx 2018-11-29: Think about if every returned node should contain - // a `cid` field or not - // it('resolver.get value within nested scope (1 level) returns cid of node traversed to', (done) => { - // resolver.get(cid2, 'Links/0/Hash/Data', (err, result) => { - // expect(err).to.not.exist() - // expect(result.cid).to.deep.equal(cid1) - // done() - // }) - // }) - - // TODO vmx 2018-11-29: remove this `get()` call with the new `get()` - // it('resolver.remove', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err, node) => { - // expect(err).to.not.exist() - // remove() - // }) - // }) - // - // function remove () { - // resolver.remove(cid1, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err) => { - // expect(err).to.exist() - // done() - // }) - // }) - // } - // }) + it('resolver.get round-trip', async () => { + const resultPut = resolver.put([node1], multicodec.DAG_PB) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const node = await resultGet.first() + // `size` is lazy, without a call to it a deep equal check would fail + const _ = node.size // eslint-disable-line no-unused-vars + expect(node).to.deep.equal(node1) + }) + + it('resolver.remove', async () => { + // TODO vmx 2018-12-12: The same repo is used for all tests, there + // seems to be some race condition with inserting and removing items. + // Hence create a unique item for this test. Though the tests + // should really be independent so that there are no race conditions. + const createNode = new Promise((resolve, reject) => { + const data = Buffer.from('a dag-pb node') + dagPB.DAGNode.create(data, (err, node) => { + if (err) { + return reject(err) + } + return resolve(node) + }) + }) + const node = await createNode + const resultPut = resolver.put([node], multicodec.DAG_PB) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const sameAsNode = await resultGet.first() + // `size` is lazy, without a call to it a deep equal check would fail + const _ = sameAsNode.size // eslint-disable-line no-unused-vars + expect(sameAsNode.data).to.deep.equal(node.data) + return remove() + + function remove () { + return new Promise((resolve, reject) => { + resolver.remove(cid, (err) => { + expect(err).to.not.exist() + const resultGet = resolver.get([cid]) + expect(resultGet.next()).to.eventually.be.rejected() + .then(() => resolve()) + .catch((err) => reject(err)) + }) + }) + } + }) }) }) } diff --git a/test/ipld-eth-block.js b/test/ipld-eth-block.js index 72f25fa..4a3c4bd 100644 --- a/test/ipld-eth-block.js +++ b/test/ipld-eth-block.js @@ -59,20 +59,6 @@ module.exports = (repo) => { resolver._put(nc.cid, nc.node, cb) }, done) }) - - // TODO vmx 2018-11-30 Change this test to use `get()`. - // it('resolver._get', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err, result) => { - // expect(err).to.not.exist() - // expect(node1.number.toString('hex')).to.eql('01') - // expect(node1.raw).to.eql(result.value.raw) - // expect(node1.hash()).to.eql(result.value.hash()) - // done() - // }) - // }) - // }) }) describe('public api', () => { @@ -98,19 +84,6 @@ module.exports = (repo) => { expect(mh.name).to.equal('keccak-512') }) - // TODO vmx 2018-11-30: Implement getting the whole object properly - // it('root path (same as get)', (done) => { - // resolver.get(cid1, '/', (err, result) => { - // expect(err).to.not.exist() - // - // ipldEthBlock.util.cid(result.value, (err, cid) => { - // expect(err).to.not.exist() - // expect(cid).to.eql(cid1) - // done() - // }) - // }) - // }) - it('resolves value within 1st node scope', async () => { const result = resolver.resolve(cid1, 'number') const node = await result.first() @@ -146,29 +119,37 @@ module.exports = (repo) => { expect(node3.value.toString('hex')).to.eql('01') }) - // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` - // it('resolver.remove', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err, result) => { - // expect(err).to.not.exist() - // const node = result.value - // expect(node1.raw).to.eql(node.raw) - // expect(node1.hash()).to.eql(node.hash()) - // remove() - // }) - // }) - // - // function remove () { - // resolver.remove(cid1, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err) => { - // expect(err).to.exist() - // done() - // }) - // }) - // } - // }) + it('resolver.get round-trip', async () => { + const resultPut = resolver.put([node1], multicodec.ETH_BLOCK) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const node = await resultGet.first() + // TODO vmx 2018-12-12: Find out why the full nodes not deep equal + expect(node.raw).to.deep.equal(node1.raw) + }) + + it('resolver.remove', async () => { + const resultPut = resolver.put([node1], multicodec.ETH_BLOCK) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const sameAsNode1 = await resultGet.first() + expect(sameAsNode1.raw).to.deep.equal(node1.raw) + return remove() + + function remove () { + return new Promise((resolve, reject) => { + resolver.remove(cid, (err) => { + expect(err).to.not.exist() + const resultGet = resolver.get([cid]) + expect(resultGet.first()).to.eventually.be.rejected() + // eslint-disable-next-line max-nested-callbacks + .then(() => resolve()) + // eslint-disable-next-line max-nested-callbacks + .catch((err) => reject(err)) + }) + }) + } + }) }) }) } diff --git a/test/ipld-git.js b/test/ipld-git.js index 1111432..a6ba834 100644 --- a/test/ipld-git.js +++ b/test/ipld-git.js @@ -155,18 +155,6 @@ module.exports = (repo) => { resolver._put(nc.cid, nc.node, cb) }, done) }) - - // TODO vmx 2018-11-30 Change this test to use `get()`. - // it('resolver._get', (done) => { - // resolver.put(blobNode, { cid: blobCid }, (err) => { - // expect(err).to.not.exist() - // resolver.get(blobCid, (err, result) => { - // expect(err).to.not.exist() - // expect(blobNode.toString('hex')).to.eql(result.value.toString('hex')) - // done() - // }) - // }) - // }) }) describe('public api', () => { @@ -192,19 +180,6 @@ module.exports = (repo) => { expect(mh.name).to.equal('sha3-512') }) - // TODO vmx 2018-11-30: Implement getting the whole object properly - // it('resolver.get empty path', (done) => { - // resolver.get(blobCid, '', (err, result) => { - // expect(err).to.not.exist() - // - // ipldGit.util.cid(result.value, (err, cid) => { - // expect(err).to.not.exist() - // expect(cid).to.eql(blobCid) - // done() - // }) - // }) - // }) - it('resolves value within 1st node scope', async () => { const result = resolver.resolve(commitCid, 'message') const node = await result.first() @@ -267,28 +242,34 @@ module.exports = (repo) => { expect(last.value).to.eql(blobNode) }) - // // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` - // it('resolver.remove', (done) => { - // resolver.put(blobNode, { cid: blobCid }, (err) => { - // expect(err).to.not.exist() - // resolver.get(blobCid, (err, result) => { - // expect(err).to.not.exist() - // const node = result.value - // expect(blobNode.toString('hex')).to.eql(node.toString('hex')) - // remove() - // }) - // }) - // - // function remove () { - // resolver.remove(blobCid, (err) => { - // expect(err).to.not.exist() - // resolver.get(blobCid, (err) => { - // expect(err).to.exist() - // done() - // }) - // }) - // } - // }) + it('resolver.get round-trip', async () => { + const resultPut = resolver.put([blobNode], multicodec.GIT_RAW) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const node = await resultGet.first() + expect(node).to.deep.equal(blobNode) + }) + + it('resolver.remove', async () => { + const resultPut = resolver.put([blobNode], multicodec.GIT_RAW) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const sameAsBlobNode = await resultGet.first() + expect(sameAsBlobNode).to.deep.equal(blobNode) + return remove() + + function remove () { + return new Promise((resolve, reject) => { + resolver.remove(cid, (err) => { + expect(err).to.not.exist() + const resultGet = resolver.get([cid]) + expect(resultGet.next()).to.eventually.be.rejected() + .then(() => resolve()) + .catch((err) => reject(err)) + }) + }) + } + }) }) }) } diff --git a/test/ipld-zcash.js b/test/ipld-zcash.js index 19b4c39..6c0c7cb 100644 --- a/test/ipld-zcash.js +++ b/test/ipld-zcash.js @@ -107,18 +107,6 @@ module.exports = (repo) => { resolver._put(nc.cid, nc.node, cb) }, done) }) - - // // TODO vmx 2018-11-30 Change this test to use `get()`. - // it('resolver._get', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err, result) => { - // expect(err).to.not.exist() - // expect(node1.version).to.eql(result.value.version) - // done() - // }) - // }) - // }) }) describe('public api', () => { @@ -144,19 +132,6 @@ module.exports = (repo) => { expect(mh.name).to.equal('sha3-512') }) - // // TODO vmx 2018-11-30: Implement getting the whole object properly - // it('root path (same as get)', (done) => { - // resolver.get(cid1, '/', (err, result) => { - // expect(err).to.not.exist() - // - // ipldZcash.util.cid(result.value, (err, cid) => { - // expect(err).to.not.exist() - // expect(cid).to.eql(cid1) - // done() - // }) - // }) - // }) - it('resolves value within 1st node scope', async () => { const result = resolver.resolve(cid1, 'version') const node = await result.first() @@ -192,27 +167,34 @@ module.exports = (repo) => { expect(node3.value).to.eql(1) }) - // // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` - // it('resolver.remove', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err, result) => { - // expect(err).to.not.exist() - // expect(result.value.version).to.eql(1) - // remove() - // }) - // }) - // - // function remove () { - // resolver.remove(cid1, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err) => { - // expect(err).to.exist() - // done() - // }) - // }) - // } - // }) + it('resolver.get round-trip', async () => { + const resultPut = resolver.put([node1], multicodec.ZCASH_BLOCK) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const node = await resultGet.first() + expect(node.toString()).to.deep.equal(node1.toString()) + }) + + it('resolver.remove', async () => { + const resultPut = resolver.put([node1], multicodec.ZCASH_BLOCK) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const sameAsNode1 = await resultGet.first() + expect(sameAsNode1).to.deep.equal(node1) + return remove() + + function remove () { + return new Promise((resolve, reject) => { + resolver.remove(cid, (err) => { + expect(err).to.not.exist() + const resultGet = resolver.get([cid]) + expect(resultGet.next()).to.eventually.be.rejected() + .then(() => resolve()) + .catch((err) => reject(err)) + }) + }) + } + }) }) }) } From b7c51016ce10f8a952b77305a197901c977718d4 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 12 Dec 2018 17:12:42 +0100 Subject: [PATCH 06/27] feat: implementation of the new `remove()` function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: `remove()` has a new API. The API docs for it: > Remove IPLD Nodes by the given `cids` - `cids` (`Iterable`): the CIDs of the IPLD Nodes that should be removed. Throws an error if any of the Blocks can’t be removed. This operation is *not* atomic, some Blocks might have already been removed. --- src/index.js | 35 +++++++++++++++++++++++++++++++++-- test/ipld-bitcoin.js | 17 +++++++---------- test/ipld-dag-cbor.js | 38 +++++++++++++++++--------------------- test/ipld-dag-pb.js | 17 +++++++---------- test/ipld-eth-block.js | 21 +++++++++------------ test/ipld-git.js | 17 +++++++---------- test/ipld-zcash.js | 17 +++++++---------- 7 files changed, 87 insertions(+), 75 deletions(-) diff --git a/src/index.js b/src/index.js index 9dfa7d6..67f5218 100644 --- a/src/index.js +++ b/src/index.js @@ -392,8 +392,39 @@ class IPLDResolver { return p } - remove (cids, callback) { - this.bs.delete(cids, callback) + /** + * Remove IPLD Nodes by the given CIDs. + * + * Throws an error if any of the Blocks can’t be removed. This operation is + * *not* atomic, some Blocks might have already been removed. + * + * @param {Iterable.} cids - The CIDs of the IPLD Nodes that should be removed + * @return {void} + */ + remove (cids) { + if (!typical.isIterable(cids) || typical.isString(cids) || + Buffer.isBuffer(cids)) { + throw new Error('`cids` must be an iterable of CIDs') + } + + const next = () => { + // End iteration if there are no more nodes to remove + if (cids.length === 0) { + return Promise.resolve({ done: true }) + } + + return new Promise((resolve, reject) => { + const cid = cids.shift() + this.bs.delete(cid, (err) => { + if (err) { + return reject(err) + } + return resolve({ done: false, value: cid }) + }) + }) + } + + return fancyIterator(next) } /* */ diff --git a/test/ipld-bitcoin.js b/test/ipld-bitcoin.js index c3c6a75..7f1d33e 100644 --- a/test/ipld-bitcoin.js +++ b/test/ipld-bitcoin.js @@ -178,16 +178,13 @@ module.exports = (repo) => { expect(sameAsNode1).to.deep.equal(node1) return remove() - function remove () { - return new Promise((resolve, reject) => { - resolver.remove(cid, (err) => { - expect(err).to.not.exist() - const resultGet = resolver.get([cid]) - expect(resultGet.next()).to.eventually.be.rejected() - .then(() => resolve()) - .catch((err) => reject(err)) - }) - }) + async function remove () { + const resultRemove = resolver.remove([cid]) + // The items are deleted through iteration + await resultRemove.last() + // Verify that the item got really deleted + const resultGet = resolver.get([cid]) + await expect(resultGet.next()).to.eventually.be.rejected() } }) }) diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index 5a5d4e1..13f80b4 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -247,27 +247,23 @@ module.exports = (repo) => { ) }) - // // TODO vmx 2018-11-30: remove this `get()` call with the new `get()` - // it('resolver.remove', (done) => { - // resolver.put(node1, { cid: cid1 }, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err, result) => { - // expect(err).to.not.exist() - // expect(node1).to.eql(result.value) - // remove() - // }) - // }) - // - // function remove () { - // resolver.remove(cid1, (err) => { - // expect(err).to.not.exist() - // resolver.get(cid1, (err) => { - // expect(err).to.exist() - // done() - // }) - // }) - // } - // }) + it('resolver.remove', async () => { + const resultPut = resolver.put([node1], multicodec.DAG_CBOR) + const cid = await resultPut.first() + const resultGet = resolver.get([cid]) + const sameAsNode1 = await resultGet.first() + expect(sameAsNode1).to.deep.equal(node1) + return remove() + + async function remove () { + const resultRemove = resolver.remove([cid]) + // The items are deleted through iteration + await resultRemove.last() + // Verify that the item got really deleted + const resultGet = resolver.get([cid]) + await expect(resultGet.next()).to.eventually.be.rejected() + } + }) }) }) } diff --git a/test/ipld-dag-pb.js b/test/ipld-dag-pb.js index 6908256..2dc23ea 100644 --- a/test/ipld-dag-pb.js +++ b/test/ipld-dag-pb.js @@ -212,16 +212,13 @@ module.exports = (repo) => { expect(sameAsNode.data).to.deep.equal(node.data) return remove() - function remove () { - return new Promise((resolve, reject) => { - resolver.remove(cid, (err) => { - expect(err).to.not.exist() - const resultGet = resolver.get([cid]) - expect(resultGet.next()).to.eventually.be.rejected() - .then(() => resolve()) - .catch((err) => reject(err)) - }) - }) + async function remove () { + const resultRemove = resolver.remove([cid]) + // The items are deleted through iteration + await resultRemove.last() + // Verify that the item got really deleted + const resultGet = resolver.get([cid]) + await expect(resultGet.next()).to.eventually.be.rejected() } }) }) diff --git a/test/ipld-eth-block.js b/test/ipld-eth-block.js index 4a3c4bd..f60aabf 100644 --- a/test/ipld-eth-block.js +++ b/test/ipld-eth-block.js @@ -2,8 +2,10 @@ 'use strict' const chai = require('chai') +const chaiAsProised = require('chai-as-promised') const dirtyChai = require('dirty-chai') const expect = chai.expect +chai.use(chaiAsProised) chai.use(dirtyChai) const BlockService = require('ipfs-block-service') const ipldEthBlock = require('ipld-ethereum').ethBlock @@ -136,18 +138,13 @@ module.exports = (repo) => { expect(sameAsNode1.raw).to.deep.equal(node1.raw) return remove() - function remove () { - return new Promise((resolve, reject) => { - resolver.remove(cid, (err) => { - expect(err).to.not.exist() - const resultGet = resolver.get([cid]) - expect(resultGet.first()).to.eventually.be.rejected() - // eslint-disable-next-line max-nested-callbacks - .then(() => resolve()) - // eslint-disable-next-line max-nested-callbacks - .catch((err) => reject(err)) - }) - }) + async function remove () { + const resultRemove = resolver.remove([cid]) + // The items are deleted through iteration + await resultRemove.last() + // Verify that the item got really deleted + const resultGet = resolver.get([cid]) + await expect(resultGet.next()).to.eventually.be.rejected() } }) }) diff --git a/test/ipld-git.js b/test/ipld-git.js index a6ba834..10de0a4 100644 --- a/test/ipld-git.js +++ b/test/ipld-git.js @@ -258,16 +258,13 @@ module.exports = (repo) => { expect(sameAsBlobNode).to.deep.equal(blobNode) return remove() - function remove () { - return new Promise((resolve, reject) => { - resolver.remove(cid, (err) => { - expect(err).to.not.exist() - const resultGet = resolver.get([cid]) - expect(resultGet.next()).to.eventually.be.rejected() - .then(() => resolve()) - .catch((err) => reject(err)) - }) - }) + async function remove () { + const resultRemove = resolver.remove([cid]) + // The items are deleted through iteration + await resultRemove.last() + // Verify that the item got really deleted + const resultGet = resolver.get([cid]) + await expect(resultGet.next()).to.eventually.be.rejected() } }) }) diff --git a/test/ipld-zcash.js b/test/ipld-zcash.js index 6c0c7cb..35a25d3 100644 --- a/test/ipld-zcash.js +++ b/test/ipld-zcash.js @@ -183,16 +183,13 @@ module.exports = (repo) => { expect(sameAsNode1).to.deep.equal(node1) return remove() - function remove () { - return new Promise((resolve, reject) => { - resolver.remove(cid, (err) => { - expect(err).to.not.exist() - const resultGet = resolver.get([cid]) - expect(resultGet.next()).to.eventually.be.rejected() - .then(() => resolve()) - .catch((err) => reject(err)) - }) - }) + async function remove () { + const resultRemove = resolver.remove([cid]) + // The items are deleted through iteration + await resultRemove.last() + // Verify that the item got really deleted + const resultGet = resolver.get([cid]) + await expect(resultGet.next()).to.eventually.be.rejected() } }) }) From 57858c9908cc9e336381472f950c899afcc3e2b9 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 13 Dec 2018 00:17:01 +0100 Subject: [PATCH 07/27] feat: implementation of the new `addFormat/removeFormat()` functions BREAKING CHANGE: They replace the `support.add()` and `support.rm()` functions. The API docs for it: `.addFormat(ipldFormatImplementation)`: > Add support for an IPLD Format - `ipldFormatImplementation` (`IPLD Format`, required): the implementation of an IPLD Format. `.removeFormat(codec)`: > Remove support for an IPLD Format - `codec` (`multicodec`, required): the codec of the IPLD Format to remove. --- src/index.js | 67 +++++++++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/src/index.js b/src/index.js index 67f5218..0b9f39f 100644 --- a/src/index.js +++ b/src/index.js @@ -29,44 +29,51 @@ class IPLDResolver { // Object with current list of active resolvers this.resolvers = {} - // API entry point - this.support = {} - - // Adds support for an IPLD format - this.support.add = (codec, resolver, util) => { - if (this.resolvers[codec]) { - const codecName = multicodec.print[codec] - throw new Error(`Resolver already exists for codec "${codecName}"`) - } - - this.resolvers[codec] = { - resolver: resolver, - util: util - } - } - if (options.loadFormat === undefined) { - this.support.load = async (codec) => { + this.loadFormat = async (codec) => { const codecName = multicodec.print[codec] throw new Error(`No resolver found for codec "${codecName}"`) } } else { - this.support.load = options.loadFormat - } - - this.support.rm = (codec) => { - if (this.resolvers[codec]) { - delete this.resolvers[codec] - } + this.loadFormat = options.loadFormat } // Enable all supplied formats for (const format of options.formats) { - const { resolver, util } = format - // IPLD Formats are using strings instead of constants for the multicodec - const codecBuffer = multicodec.getCodeVarint(resolver.multicodec) - const codec = multicodec.getCode(codecBuffer) - this.support.add(codec, resolver, util) + this.addFormat(format) + } + } + + /** + * Add support for an IPLD Format. + * + * @param {Object} format - The implementation of an IPLD Format. + * @returns {void} + */ + addFormat (format) { + // IPLD Formats are using strings instead of constants for the multicodec + const codecBuffer = multicodec.getCodeVarint(format.resolver.multicodec) + const codec = multicodec.getCode(codecBuffer) + if (this.resolvers[codec]) { + const codecName = multicodec.print[codec] + throw new Error(`Resolver already exists for codec "${codecName}"`) + } + + this.resolvers[codec] = { + resolver: format.resolver, + util: format.util + } + } + + /** + * Remove support for an IPLD Format. + * + * @param {number} codec - The codec of the IPLD Format to remove. + * @returns {void} + */ + removeFormat (codec) { + if (this.resolvers[codec]) { + delete this.resolvers[codec] } } @@ -443,7 +450,7 @@ class IPLDResolver { } // If not supported, attempt to dynamically load this format - const format = await this.support.load(codec) + const format = await this.loadFormat(codec) this.resolvers[codec] = format return format } From 4e0bdea9e358fe82c08aacd65410e14fd7738e93 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 13 Dec 2018 00:26:26 +0100 Subject: [PATCH 08/27] feat: implementation of the new `tree()` function BREAKING CHANGE: This replaces the `treeStream()` function. The API docs for it: > Returns all the paths that can be resolved into. - `cid` (`CID`, required): the CID to get the paths from. - `path` (`IPLD Path`, default: ''): the path to start to retrieve the other paths from. - `options`: - `recursive` (`bool`, default: false): whether to get the paths recursively or not. `false` resolves only the paths of the given CID. Returns an async iterator of all the paths (as Strings) you could resolve into. --- package.json | 3 - src/index.js | 244 +++++++++++++++++++++--------------------- test/basics.js | 14 +-- test/ipld-dag-cbor.js | 98 +++++++---------- 4 files changed, 164 insertions(+), 195 deletions(-) diff --git a/package.json b/package.json index 7c7306f..34445eb 100644 --- a/package.json +++ b/package.json @@ -62,9 +62,6 @@ "ipld-raw": "^2.0.1", "merge-options": "^1.0.1", "multicodec": "~0.5.0", - "pull-defer": "~0.2.3", - "pull-stream": "^3.6.9", - "pull-traverse": "^1.0.3", "typical": "^3.0.0" }, "contributors": [ diff --git a/src/index.js b/src/index.js index 0b9f39f..2fcf8b6 100644 --- a/src/index.js +++ b/src/index.js @@ -1,11 +1,7 @@ 'use strict' const Block = require('ipfs-block') -const pull = require('pull-stream') const CID = require('cids') -const pullDeferSource = require('pull-defer').source -const pullTraverse = require('pull-traverse') -const map = require('async/map') const waterfall = require('async/waterfall') const mergeOptions = require('merge-options') const ipldDagCbor = require('ipld-dag-cbor') @@ -283,122 +279,6 @@ class IPLDResolver { return fancyIterator(next) } - treeStream (cid, path, options) { - if (typeof path === 'object') { - options = path - path = undefined - } - - options = options || {} - - let p - - if (!options.recursive) { - p = pullDeferSource() - - waterfall([ - async () => { - return this._getFormat(cid.codec) - }, - (format, cb) => this.bs.get(cid, (err, block) => { - if (err) return cb(err) - cb(null, format, block) - }), - (format, block, cb) => format.resolver.tree(block.data, cb) - ], (err, paths) => { - if (err) { - p.abort(err) - return p - } - p.resolve(pull.values(paths)) - }) - } - - // recursive - if (options.recursive) { - p = pull( - pullTraverse.widthFirst({ - basePath: null, - cid: cid - }, (el) => { - // pass the paths through the pushable pull stream - // continue traversing the graph by returning - // the next cids with deferred - - if (typeof el === 'string') { - return pull.empty() - } - - const deferred = pullDeferSource() - const cid = el.cid - - waterfall([ - async () => { - return this._getFormat(cid.codec) - }, - (format, cb) => this.bs.get(cid, (err, block) => { - if (err) return cb(err) - cb(null, format, block) - }), - (format, block, cb) => format.resolver.tree(block.data, (err, paths) => { - if (err) { - return cb(err) - } - map(paths, (p, cb) => { - format.resolver.isLink(block.data, p, (err, link) => { - if (err) { - return cb(err) - } - cb(null, { path: p, link: link }) - }) - }, cb) - }) - ], (err, paths) => { - if (err) { - deferred.abort(err) - return deferred - } - - deferred.resolve(pull.values(paths.map((p) => { - const base = el.basePath ? el.basePath + '/' + p.path : p.path - if (p.link) { - return { - basePath: base, - cid: IPLDResolver._maybeCID(p.link) - } - } - return base - }))) - }) - return deferred - }), - pull.map((e) => { - if (typeof e === 'string') { - return e - } - return e.basePath - }), - pull.filter(Boolean) - ) - } - - // filter out by path - if (path) { - return pull( - p, - pull.map((el) => { - if (el.indexOf(path) === 0) { - el = el.slice(path.length + 1) - return el - } - }), - pull.filter(Boolean) - ) - } - - return p - } - /** * Remove IPLD Nodes by the given CIDs. * @@ -434,6 +314,130 @@ class IPLDResolver { return fancyIterator(next) } + /** + * Returns all the paths that can be resolved into. + * + * @param {Object} cid - The ID to get the paths from + * @param {string} [offsetPath=''] - the path to start to retrieve the other paths from. + * @param {Object} [userOptions] + * @param {number} [userOptions.recursive=false] - whether to get the paths recursively or not. `false` resolves only the paths of the given CID. + * @returns {Iterable.>} - Returns an async iterator with paths that can be resolved into + */ + tree (cid, offsetPath, userOptions) { + if (typeof offsetPath === 'object') { + userOptions = offsetPath + offsetPath = undefined + } + offsetPath = offsetPath || '' + + const defaultOptions = { + recursive: false + } + const options = mergeOptions(defaultOptions, userOptions) + + // Get available paths from a block + const getPaths = (cid) => { + return new Promise(async (resolve, reject) => { + let format + try { + format = await this._getFormat(cid.codec) + } catch (error) { + return reject(error) + } + this.bs.get(cid, (err, block) => { + if (err) { + return reject(err) + } + format.resolver.tree(block.data, (err, paths) => { + if (err) { + return reject(err) + } + return resolve({ paths, block }) + }) + }) + }) + } + + // If a path is a link then follow it and return its CID + const maybeRecurse = (block, treePath) => { + return new Promise(async (resolve, reject) => { + // A treepath we might want to follow recursively + const format = await this._getFormat(block.cid.codec) + format.resolver.isLink(block.data, treePath, (err, link) => { + if (err) { + return reject(err) + } + // Something to follow recusively, hence push it into the queue + if (link) { + const cid = IPLDResolver._maybeCID(link) + resolve(cid) + } else { + resolve(null) + } + }) + }) + } + + // The list of paths that will get returned + let treePaths = [] + // The current block, needed to call `isLink()` on every interation + let block + // The list of items we want to follow recursively. The items are + // an object consisting of the CID and the currently already resolved + // path + const queue = [{ cid, basePath: '' }] + // The path that was already traversed + let basePath + + const next = () => { + // End of iteration if there aren't any paths left to return or + // if we don't want to traverse recursively and have already + // returne the first level + if (treePaths.length === 0 && queue.length === 0) { + return { done: true } + } + + return new Promise(async (resolve, reject) => { + // There aren't any paths left, get them from the given CID + if (treePaths.length === 0 && queue.length > 0) { + ({ cid, basePath } = queue.shift()) + + let paths + try { + ({ block, paths } = await getPaths(cid)) + } catch (error) { + return reject(error) + } + treePaths.push(...paths) + } + const treePath = treePaths.shift() + let fullPath = basePath + treePath + + // Only follow links if recursion is intended + if (options.recursive) { + cid = await maybeRecurse(block, treePath) + if (cid !== null) { + queue.push({ cid, basePath: fullPath + '/' }) + } + } + + // Return it if it matches the given offset path, but is not the + // offset path itself + if (fullPath.startsWith(offsetPath) && + fullPath.length > offsetPath.length) { + if (offsetPath.length > 0) { + fullPath = fullPath.slice(offsetPath.length + 1) + } + return resolve({ done: false, value: fullPath }) + } else { // Else move on to the next iteration before returning + return resolve(next()) + } + }) + } + + return fancyIterator(next) + } + /* */ /* internals */ /* */ diff --git a/test/basics.js b/test/basics.js index a6d0eaa..f0fd58f 100644 --- a/test/basics.js +++ b/test/basics.js @@ -11,7 +11,6 @@ const BlockService = require('ipfs-block-service') const CID = require('cids') const multihash = require('multihashes') const multicodec = require('multicodec') -const pull = require('pull-stream') const inMemory = require('ipld-in-memory') const IPLDResolver = require('../src') @@ -81,7 +80,7 @@ module.exports = (repo) => { }) }) - it('treeStream - errors on unknown resolver', (done) => { + it('tree - errors on unknown resolver', async () => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) // choosing a format that is not supported @@ -90,14 +89,9 @@ module.exports = (repo) => { 'blake2b-8', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1') ) - pull( - r.treeStream(cid, '/', {}), - pull.collect(function (err) { - expect(err).to.exist() - expect(err.message).to.eql('No resolver found for codec "blake2b-8"') - done() - }) - ) + const result = r.tree(cid) + await expect(result.next()).to.be.rejectedWith( + 'No resolver found for codec "blake2b-8"') }) }) } diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index 13f80b4..11e2766 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -11,7 +11,6 @@ const BlockService = require('ipfs-block-service') const dagCBOR = require('ipld-dag-cbor') const series = require('async/series') const each = require('async/each') -const pull = require('pull-stream') const multicodec = require('multicodec') const multihash = require('multihashes') @@ -176,75 +175,50 @@ module.exports = (repo) => { expect(node).to.deep.equal(node1) }) - it('resolver.tree', (done) => { - pull( - resolver.treeStream(cid3), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([ - 'one', - 'two', - 'someData' - ]) - done() - }) - ) + it('resolver.tree', async () => { + const result = resolver.tree(cid3) + const paths = await result.all() + expect(paths).to.eql([ + 'one', + 'two', + 'someData' + ]) }) - it('resolver.tree with exist()ent path', (done) => { - pull( - resolver.treeStream(cid3, 'one'), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([]) - done() - }) - ) + it('resolver.tree with exist()ent path', async () => { + const result = resolver.tree(cid3, 'one') + const paths = await result.all() + expect(paths).to.eql([]) }) - it('resolver.tree with non exist()ent path', (done) => { - pull( - resolver.treeStream(cid3, 'bananas'), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([]) - done() - }) - ) + it('resolver.tree with non exist()ent path', async () => { + const result = resolver.tree(cid3, 'bananas') + const paths = await result.all() + expect(paths).to.eql([]) }) - it('resolver.tree recursive', (done) => { - pull( - resolver.treeStream(cid3, { recursive: true }), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([ - 'one', - 'two', - 'someData', - 'one/someData', - 'two/one', - 'two/someData', - 'two/one/someData' - ]) - done() - }) - ) + it('resolver.tree recursive', async () => { + const result = resolver.tree(cid3, { recursive: true }) + const paths = await result.all() + expect(paths).to.eql([ + 'one', + 'two', + 'someData', + 'one/someData', + 'two/one', + 'two/someData', + 'two/one/someData' + ]) }) - it('resolver.tree with exist()ent path recursive', (done) => { - pull( - resolver.treeStream(cid3, 'two', { recursive: true }), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([ - 'one', - 'someData', - 'one/someData' - ]) - done() - }) - ) + it('resolver.tree with exist()ent path recursive', async () => { + const result = resolver.tree(cid3, 'two', { recursive: true }) + const paths = await result.all() + expect(paths).to.eql([ + 'one', + 'someData', + 'one/someData' + ]) }) it('resolver.remove', async () => { From 0c329d3b8cdb231453e72fe74139d8d4b8c5b848 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 7 Feb 2019 17:57:01 +0100 Subject: [PATCH 09/27] chore: update dependencies --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 34445eb..fd62be8 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "license": "MIT", "devDependencies": { "aegir": "^18.2.1", - "bitcoinjs-lib": "^4.0.2", + "bitcoinjs-lib": "^4.0.3", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "dirty-chai": "^2.0.1", From f34aa41be46f3f90a60b924c1fd63ca3318e4b11 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 20 Feb 2019 16:37:22 +0100 Subject: [PATCH 10/27] fix: error if loadFormat() is not a function Previously it was only erroring when it was undefined. --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 2fcf8b6..6965f87 100644 --- a/src/index.js +++ b/src/index.js @@ -25,7 +25,7 @@ class IPLDResolver { // Object with current list of active resolvers this.resolvers = {} - if (options.loadFormat === undefined) { + if (typeof options.loadFormat !== 'function') { this.loadFormat = async (codec) => { const codecName = multicodec.print[codec] throw new Error(`No resolver found for codec "${codecName}"`) From 8aabe300f3ca262bd5a26f830e2eea639f50c1f2 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 20 Feb 2019 16:39:00 +0100 Subject: [PATCH 11/27] fix: add dynamically loaded format via addFormat() The return value of `loadFormat()` needs to have the same type as the format you pass in via `addFormat()`. Hence use the `addFormat()` code path in order to add a new format dynamically. --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 6965f87..6096062 100644 --- a/src/index.js +++ b/src/index.js @@ -455,7 +455,7 @@ class IPLDResolver { // If not supported, attempt to dynamically load this format const format = await this.loadFormat(codec) - this.resolvers[codec] = format + this.addFormat(format) return format } From eae187007902e6be1854aac2a2601f4ead42e92b Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 20 Feb 2019 16:41:11 +0100 Subject: [PATCH 12/27] feat: make addFormat() and removeFormat() return the instance This way you can chain addFormat()/removeFormat() calls. --- src/index.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index 6096062..83c430f 100644 --- a/src/index.js +++ b/src/index.js @@ -44,7 +44,7 @@ class IPLDResolver { * Add support for an IPLD Format. * * @param {Object} format - The implementation of an IPLD Format. - * @returns {void} + * @returns {this} */ addFormat (format) { // IPLD Formats are using strings instead of constants for the multicodec @@ -59,18 +59,22 @@ class IPLDResolver { resolver: format.resolver, util: format.util } + + return this } /** * Remove support for an IPLD Format. * * @param {number} codec - The codec of the IPLD Format to remove. - * @returns {void} + * @returns {this} */ removeFormat (codec) { if (this.resolvers[codec]) { delete this.resolvers[codec] } + + return this } /** From c0f49f8418c1e5cb3584aa059c2ec82b8b093d83 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 20 Feb 2019 16:44:19 +0100 Subject: [PATCH 13/27] chore: fix typo in comment --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 83c430f..81e39d2 100644 --- a/src/index.js +++ b/src/index.js @@ -173,7 +173,7 @@ class IPLDResolver { return new Promise(async (resolve, reject) => { // Lazy load block. - // Currntly the BlockService return all nodes as an array. In the + // Currently the BlockService return all nodes as an array. In the // future this will also be an iterator if (blocks === undefined) { const cidsArray = Array.from(cids) From 7678a2b678592ef12d39d7d0fdb3ba6cd916cea3 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 20 Feb 2019 17:31:58 +0100 Subject: [PATCH 14/27] refactor: promisify resolve() Instead of mixing callbacks and Promises, use `promisify()`. --- src/index.js | 74 ++++++++++++++++++++-------------------------------- 1 file changed, 28 insertions(+), 46 deletions(-) diff --git a/src/index.js b/src/index.js index 81e39d2..66593d7 100644 --- a/src/index.js +++ b/src/index.js @@ -1,5 +1,7 @@ 'use strict' +const promisify = require('util').promisify + const Block = require('ipfs-block') const CID = require('cids') const waterfall = require('async/waterfall') @@ -94,57 +96,37 @@ class IPLDResolver { throw new Error('`path` argument must be a string') } - const next = () => { + const next = async () => { // End iteration if there isn't a CID to follow anymore if (cid === null) { - return Promise.resolve({ done: true }) + return { done: true } } - return new Promise(async (resolve, reject) => { - let format - try { - format = await this._getFormat(cid.codec) - } catch (err) { - return reject(err) - } - - // get block - // use local resolver - // update path value - this.bs.get(cid, (err, block) => { - if (err) { - return reject(err) - } - - format.resolver.resolve(block.data, path, (err, result) => { - if (err) { - return reject(err) - } - - // Prepare for the next iteration if there is a `remainderPath` - path = result.remainderPath - let value = result.value - // NOTE vmx 2018-11-29: Not all IPLD Formats return links as - // CIDs yet. Hence try to convert old style links to CIDs - if (Object.keys(value).length === 1 && '/' in value) { - value = new CID(value['/']) - } - if (CID.isCID(value)) { - cid = value - } else { - cid = null - } + const format = await this._getFormat(cid.codec) + + // get block + // use local resolver + // update path value + const block = await promisify(this.bs.get.bind(this.bs))(cid) + const result = await promisify(format.resolver.resolve)(block.data, path) + + // Prepare for the next iteration if there is a `remainderPath` + path = result.remainderPath + let value = result.value + // NOTE vmx 2018-11-29: Not all IPLD Formats return links as + // CIDs yet. Hence try to convert old style links to CIDs + if (Object.keys(value).length === 1 && '/' in value) { + value = new CID(value['/']) + } + cid = CID.isCID(value) ? value : null - return resolve({ - done: false, - value: { - remainderPath: path, - value - } - }) - }) - }) - }) + return { + done: false, + value: { + remainderPath: path, + value + } + } } return fancyIterator(next) From 63585361cf68a2e1995c87d72392519324a37dac Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 20 Feb 2019 17:39:13 +0100 Subject: [PATCH 15/27] refactor: promisify get() Instead of mixing callbacks and Promises, use `promisify()`. --- src/index.js | 63 +++++++++++++++------------------------------------- 1 file changed, 18 insertions(+), 45 deletions(-) diff --git a/src/index.js b/src/index.js index 66593d7..e219595 100644 --- a/src/index.js +++ b/src/index.js @@ -145,43 +145,28 @@ class IPLDResolver { } let blocks - const next = () => { + const next = async () => { // End of iteration if there aren't any blocks left to return if (cids.length === 0 || (blocks !== undefined && blocks.length === 0) ) { - return Promise.resolve({ done: true }) + return { done: true } } - return new Promise(async (resolve, reject) => { - // Lazy load block. - // Currently the BlockService return all nodes as an array. In the - // future this will also be an iterator - if (blocks === undefined) { - const cidsArray = Array.from(cids) - this.bs.getMany(cidsArray, async (err, returnedBlocks) => { - if (err) { - return reject(err) - } - blocks = returnedBlocks - const block = blocks.shift() - try { - const node = await this._deserialize(block) - return resolve({ done: false, value: node }) - } catch (err) { - return reject(err) - } - }) - } else { - const block = blocks.shift() - try { - const node = await this._deserialize(block) - return resolve({ done: false, value: node }) - } catch (err) { - return reject(err) - } - } - }) + // Lazy load block. + // Currently the BlockService return all nodes as an array. In the + // future this will also be an iterator + if (blocks === undefined) { + const cidsArray = Array.from(cids) + blocks = await promisify(this.bs.getMany.bind(this.bs))(cidsArray) + } + const block = blocks.shift() + const node = await this._deserialize(block) + + return { + done: false, + value: node + } } return fancyIterator(next) @@ -472,20 +457,8 @@ class IPLDResolver { * @return {Object} = Returns the deserialized node */ async _deserialize (block) { - return new Promise((resolve, reject) => { - this._getFormat(block.cid.codec).then((format) => { - // TODO vmx 2018-12-11: Make this one async/await once - // `util.serialize()` is a Promise - format.util.deserialize(block.data, (err, deserialized) => { - if (err) { - return reject(err) - } - return resolve(deserialized) - }) - }).catch((err) => { - return reject(err) - }) - }) + const format = await this._getFormat(block.cid.codec) + return promisify(format.util.deserialize)(block.data) } /** From 2f286a97c52e303cfceb1bf560a468f119fd9550 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 20 Feb 2019 18:02:20 +0100 Subject: [PATCH 16/27] refactor: promisify put() Instead of mixing callbacks and Promises, use `promisify()`. --- package.json | 20 ++++----- src/index.js | 94 ++++++++++++++---------------------------- test/basics.js | 16 ------- test/ipld-bitcoin.js | 13 ------ test/ipld-dag-cbor.js | 13 ------ test/ipld-dag-pb.js | 13 ------ test/ipld-eth-block.js | 13 ------ test/ipld-git.js | 15 ------- test/ipld-zcash.js | 13 ------ 9 files changed, 42 insertions(+), 168 deletions(-) diff --git a/package.json b/package.json index fd62be8..dda3656 100644 --- a/package.json +++ b/package.json @@ -34,30 +34,30 @@ "license": "MIT", "devDependencies": { "aegir": "^18.2.1", + "async": "^2.6.1", "bitcoinjs-lib": "^4.0.3", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "dirty-chai": "^2.0.1", - "ethereumjs-block": "^2.1.0", + "ethereumjs-block": "^2.2.0", "ipfs-block-service": "~0.15.2", "ipfs-repo": "~0.26.1", - "ipld-bitcoin": "~0.1.8", - "ipld-ethereum": "^2.0.1", - "ipld-git": "~0.2.2", + "ipld-bitcoin": "~0.1.9", + "ipld-ethereum": "^2.0.3", + "ipld-git": "~0.2.3", "ipld-in-memory": "^2.0.0", "ipld-zcash": "~0.1.6", - "merkle-patricia-tree": "^2.3.2", + "merkle-patricia-tree": "^3.0.0", "multihashes": "~0.4.14", "ncp": "^2.0.0", - "rimraf": "^2.6.2", - "rlp": "^2.1.0", + "rimraf": "^2.6.3", + "rlp": "^2.2.2", "zcash-bitcore-lib": "~0.13.20-rc3" }, "dependencies": { - "async": "^2.6.1", - "cids": "~0.5.5", + "cids": "~0.5.7", "ipfs-block": "~0.8.0", - "ipld-dag-cbor": "~0.13.0", + "ipld-dag-cbor": "~0.13.1", "ipld-dag-pb": "~0.15.2", "ipld-raw": "^2.0.1", "merge-options": "^1.0.1", diff --git a/src/index.js b/src/index.js index e219595..5699c29 100644 --- a/src/index.js +++ b/src/index.js @@ -4,7 +4,6 @@ const promisify = require('util').promisify const Block = require('ipfs-block') const CID = require('cids') -const waterfall = require('async/waterfall') const mergeOptions = require('merge-options') const ipldDagCbor = require('ipld-dag-cbor') const ipldDagPb = require('ipld-dag-pb') @@ -13,8 +12,6 @@ const multicodec = require('multicodec') const typical = require('typical') const { fancyIterator } = require('./util') -function noop () {} - class IPLDResolver { constructor (userOptions) { const options = mergeOptions(IPLDResolver.defaultOptions, userOptions) @@ -198,53 +195,39 @@ class IPLDResolver { let options let formatImpl - const next = () => { + const next = async () => { // End iteration if there are no more nodes to put if (nodes.length === 0) { - return Promise.resolve({ done: true }) + return { done: true } } - return new Promise(async (resolve, reject) => { - // Lazy load the options not when the iterator is initialized, but - // when we hit the first iteration. This way the constructor can be - // a synchronous function. - if (options === undefined) { - try { - formatImpl = await this._getFormat(format) - } catch (err) { - return reject(err) - } - const defaultOptions = { - hashAlg: formatImpl.defaultHashAlg, - cidVersion: 1, - onlyHash: false - } - options = mergeOptions(defaultOptions, userOptions) - } - - const node = nodes.shift() - const cidOptions = { - version: options.cidVersion, - hashAlg: options.hashAlg, - onlyHash: options.onlyHash + // Lazy load the options not when the iterator is initialized, but + // when we hit the first iteration. This way the constructor can be + // a synchronous function. + if (options === undefined) { + formatImpl = await this._getFormat(format) + const defaultOptions = { + hashAlg: formatImpl.defaultHashAlg, + cidVersion: 1, + onlyHash: false } - formatImpl.util.cid(node, cidOptions, (err, cid) => { - if (err) { - return reject(err) - } - - if (options.onlyHash) { - return resolve({ done: false, value: cid }) - } + options = mergeOptions(defaultOptions, userOptions) + } - this._put(cid, node, (err, cid) => { - if (err) { - return reject(err) - } - return resolve({ done: false, value: cid }) - }) - }) - }) + const node = nodes.shift() + const cidOptions = { + version: options.cidVersion, + hashAlg: options.hashAlg, + onlyHash: options.onlyHash + } + const cid = await promisify(formatImpl.util.cid)(node, cidOptions) + if (!options.onlyHash) { + await this._store(cid, node) + } + return { + done: false, + value: cid + } } return fancyIterator(next) @@ -430,24 +413,11 @@ class IPLDResolver { return format } - _put (cid, node, callback) { - callback = callback || noop - - waterfall([ - (cb) => { - this._getFormat(cid.codec).then( - (format) => cb(null, format), - (error) => cb(error) - ) - }, - (format, cb) => format.util.serialize(node, cb), - (buf, cb) => this.bs.put(new Block(buf, cid), cb) - ], (err) => { - if (err) { - return callback(err) - } - callback(null, cid) - }) + async _store (cid, node) { + const format = await this._getFormat(cid.codec) + const serialized = await promisify(format.util.serialize)(node) + const block = new Block(serialized, cid) + await promisify(this.bs.put.bind(this.bs))(block) } /** diff --git a/test/basics.js b/test/basics.js index f0fd58f..4034a1c 100644 --- a/test/basics.js +++ b/test/basics.js @@ -64,22 +64,6 @@ module.exports = (repo) => { expect(() => r.put([null])).to.be.throw('`put` requires a format') }) - it('_put - errors on unknown resolver', (done) => { - const bs = new BlockService(repo) - const r = new IPLDResolver({ blockService: bs }) - // choosing a format that is not supported - const cid = new CID( - 1, - 'blake2b-8', - multihash.encode(Buffer.from('abcd', 'hex'), 'sha1') - ) - r._put(cid, null, (err, result) => { - expect(err).to.exist() - expect(err.message).to.eql('No resolver found for codec "blake2b-8"') - done() - }) - }) - it('tree - errors on unknown resolver', async () => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) diff --git a/test/ipld-bitcoin.js b/test/ipld-bitcoin.js index 7f1d33e..f7bd238 100644 --- a/test/ipld-bitcoin.js +++ b/test/ipld-bitcoin.js @@ -10,7 +10,6 @@ const ipldBitcoin = require('ipld-bitcoin') const BitcoinBlock = require('bitcoinjs-lib').Block const multihash = require('multihashes') const series = require('async/series') -const each = require('async/each') const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -92,18 +91,6 @@ module.exports = (repo) => { } }) - describe('internals', () => { - it('resolver._put', (done) => { - each([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ], (nc, cb) => { - resolver._put(nc.cid, nc.node, cb) - }, done) - }) - }) - describe('public api', () => { it('resolver.put with format', async () => { const result = resolver.put([node1], multicodec.BITCOIN_BLOCK) diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index 11e2766..b9f8cb0 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -10,7 +10,6 @@ chai.use(chaiAsProised) const BlockService = require('ipfs-block-service') const dagCBOR = require('ipld-dag-cbor') const series = require('async/series') -const each = require('async/each') const multicodec = require('multicodec') const multihash = require('multihashes') @@ -78,18 +77,6 @@ module.exports = (repo) => { } }) - describe('internals', () => { - it('resolver._put', (done) => { - each([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ], (nc, cb) => { - resolver._put(nc.cid, nc.node, cb) - }, done) - }) - }) - describe('public api', () => { it('resolver.put with format', async () => { const result = resolver.put([node1], multicodec.DAG_CBOR) diff --git a/test/ipld-dag-pb.js b/test/ipld-dag-pb.js index 2dc23ea..ed7a5fc 100644 --- a/test/ipld-dag-pb.js +++ b/test/ipld-dag-pb.js @@ -10,7 +10,6 @@ chai.use(dirtyChai) const BlockService = require('ipfs-block-service') const dagPB = require('ipld-dag-pb') const series = require('async/series') -const each = require('async/each') const multihash = require('multihashes') const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -108,18 +107,6 @@ module.exports = (repo) => { } }) - describe('internals', () => { - it('resolver._put', (done) => { - each([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ], (nc, cb) => { - resolver._put(nc.cid, nc.node, cb) - }, done) - }) - }) - describe('public api', () => { it('resolver.put with format', async () => { const result = resolver.put([node1], multicodec.DAG_PB) diff --git a/test/ipld-eth-block.js b/test/ipld-eth-block.js index f60aabf..ddf9ab7 100644 --- a/test/ipld-eth-block.js +++ b/test/ipld-eth-block.js @@ -11,7 +11,6 @@ const BlockService = require('ipfs-block-service') const ipldEthBlock = require('ipld-ethereum').ethBlock const EthBlockHeader = require('ethereumjs-block/header') const multihash = require('multihashes') -const each = require('async/each') const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -51,18 +50,6 @@ module.exports = (repo) => { ;[cid1, cid2, cid3] = await result.all() }) - describe('internals', () => { - it('resolver._put', (done) => { - each([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ], (nc, cb) => { - resolver._put(nc.cid, nc.node, cb) - }, done) - }) - }) - describe('public api', () => { it('resolver.put with format', async () => { const result = resolver.put([node1], multicodec.ETH_BLOCK) diff --git a/test/ipld-git.js b/test/ipld-git.js index 10de0a4..15549c5 100644 --- a/test/ipld-git.js +++ b/test/ipld-git.js @@ -9,7 +9,6 @@ const BlockService = require('ipfs-block-service') const ipldGit = require('ipld-git') const multihash = require('multihashes') const series = require('async/series') -const each = require('async/each') const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -143,20 +142,6 @@ module.exports = (repo) => { } }) - describe('internals', () => { - it('resolver._put', (done) => { - each([ - { node: blobNode, cid: blobCid }, - { node: treeNode, cid: treeCid }, - { node: commitNode, cid: commitCid }, - { node: commit2Node, cid: commit2Cid }, - { node: tagNode, cid: tagCid } - ], (nc, cb) => { - resolver._put(nc.cid, nc.node, cb) - }, done) - }) - }) - describe('public api', () => { it('resolver.put with format', async () => { const result = resolver.put([blobNode], multicodec.GIT_RAW) diff --git a/test/ipld-zcash.js b/test/ipld-zcash.js index 35a25d3..a966e32 100644 --- a/test/ipld-zcash.js +++ b/test/ipld-zcash.js @@ -10,7 +10,6 @@ const ipldZcash = require('ipld-zcash') const ZcashBlockHeader = require('zcash-bitcore-lib').BlockHeader const multihash = require('multihashes') const series = require('async/series') -const each = require('async/each') const multicodec = require('multicodec') const IPLDResolver = require('../src') @@ -97,18 +96,6 @@ module.exports = (repo) => { } }) - describe('internals', () => { - it('resolver._put', (done) => { - each([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ], (nc, cb) => { - resolver._put(nc.cid, nc.node, cb) - }, done) - }) - }) - describe('public api', () => { it('resolver.put with format', async () => { const result = resolver.put([node1], multicodec.ZCASH_BLOCK) From ba6225cbcd5c3c4ed41af555dd177be994e27a4b Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 21 Feb 2019 14:18:50 +0100 Subject: [PATCH 17/27] refactor: promisify remove() Instead of mixing callbacks and Promises, use `promisify()`. --- src/index.js | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/index.js b/src/index.js index 5699c29..becfd2b 100644 --- a/src/index.js +++ b/src/index.js @@ -248,21 +248,19 @@ class IPLDResolver { throw new Error('`cids` must be an iterable of CIDs') } - const next = () => { + const next = async () => { // End iteration if there are no more nodes to remove if (cids.length === 0) { - return Promise.resolve({ done: true }) + return { done: true } } - return new Promise((resolve, reject) => { - const cid = cids.shift() - this.bs.delete(cid, (err) => { - if (err) { - return reject(err) - } - return resolve({ done: false, value: cid }) - }) - }) + const cid = cids.shift() + await promisify(this.bs.delete.bind(this.bs))(cid) + + return { + done: false, + value: cid + } } return fancyIterator(next) From a49c0261d4921f321d3569f71fe257cece8cdbf4 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 21 Feb 2019 15:41:09 +0100 Subject: [PATCH 18/27] refactor: promisify tree() Instead of mixing callbacks and Promises, use `promisify()`. --- src/index.js | 119 +++++++++++++++++++-------------------------------- 1 file changed, 45 insertions(+), 74 deletions(-) diff --git a/src/index.js b/src/index.js index becfd2b..cc7609a 100644 --- a/src/index.js +++ b/src/index.js @@ -287,47 +287,19 @@ class IPLDResolver { } const options = mergeOptions(defaultOptions, userOptions) - // Get available paths from a block - const getPaths = (cid) => { - return new Promise(async (resolve, reject) => { - let format - try { - format = await this._getFormat(cid.codec) - } catch (error) { - return reject(error) - } - this.bs.get(cid, (err, block) => { - if (err) { - return reject(err) - } - format.resolver.tree(block.data, (err, paths) => { - if (err) { - return reject(err) - } - return resolve({ paths, block }) - }) - }) - }) - } - // If a path is a link then follow it and return its CID - const maybeRecurse = (block, treePath) => { - return new Promise(async (resolve, reject) => { - // A treepath we might want to follow recursively - const format = await this._getFormat(block.cid.codec) - format.resolver.isLink(block.data, treePath, (err, link) => { - if (err) { - return reject(err) - } - // Something to follow recusively, hence push it into the queue - if (link) { - const cid = IPLDResolver._maybeCID(link) - resolve(cid) - } else { - resolve(null) - } - }) - }) + const maybeRecurse = async (block, treePath) => { + // A treepath we might want to follow recursively + const format = await this._getFormat(block.cid.codec) + const link = await promisify( + format.resolver.isLink)(block.data, treePath) + // Something to follow recusively, hence push it into the queue + if (link) { + const cid = IPLDResolver._maybeCID(link) + return cid + } else { + return null + } } // The list of paths that will get returned @@ -341,7 +313,7 @@ class IPLDResolver { // The path that was already traversed let basePath - const next = () => { + const next = async () => { // End of iteration if there aren't any paths left to return or // if we don't want to traverse recursively and have already // returne the first level @@ -349,42 +321,41 @@ class IPLDResolver { return { done: true } } - return new Promise(async (resolve, reject) => { - // There aren't any paths left, get them from the given CID - if (treePaths.length === 0 && queue.length > 0) { - ({ cid, basePath } = queue.shift()) - - let paths - try { - ({ block, paths } = await getPaths(cid)) - } catch (error) { - return reject(error) - } - treePaths.push(...paths) - } - const treePath = treePaths.shift() - let fullPath = basePath + treePath - - // Only follow links if recursion is intended - if (options.recursive) { - cid = await maybeRecurse(block, treePath) - if (cid !== null) { - queue.push({ cid, basePath: fullPath + '/' }) - } + // There aren't any paths left, get them from the given CID + if (treePaths.length === 0 && queue.length > 0) { + ({ cid, basePath } = queue.shift()) + const format = await this._getFormat(cid.codec) + block = await promisify(this.bs.get.bind(this.bs))(cid) + + const paths = await promisify(format.resolver.tree)(block.data) + treePaths.push(...paths) + } + + const treePath = treePaths.shift() + let fullPath = basePath + treePath + + // Only follow links if recursion is intended + if (options.recursive) { + cid = await maybeRecurse(block, treePath) + if (cid !== null) { + queue.push({ cid, basePath: fullPath + '/' }) } + } - // Return it if it matches the given offset path, but is not the - // offset path itself - if (fullPath.startsWith(offsetPath) && - fullPath.length > offsetPath.length) { - if (offsetPath.length > 0) { - fullPath = fullPath.slice(offsetPath.length + 1) - } - return resolve({ done: false, value: fullPath }) - } else { // Else move on to the next iteration before returning - return resolve(next()) + // Return it if it matches the given offset path, but is not the + // offset path itself + if (fullPath.startsWith(offsetPath) && + fullPath.length > offsetPath.length) { + if (offsetPath.length > 0) { + fullPath = fullPath.slice(offsetPath.length + 1) + } + return { + done: false, + value: fullPath } - }) + } else { // Else move on to the next iteration before returning + return next() + } } return fancyIterator(next) From effde6e36f0c980e87619274da65eeda9094a461 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 21 Feb 2019 17:32:38 +0100 Subject: [PATCH 19/27] refactor: make put() a generator Instead of manually coding an iterator, use an ES2015 generator. --- src/index.js | 59 +++++++++++++++++++++++----------------------------- src/util.js | 7 +++++++ 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/index.js b/src/index.js index cc7609a..a2af861 100644 --- a/src/index.js +++ b/src/index.js @@ -10,7 +10,7 @@ const ipldDagPb = require('ipld-dag-pb') const ipldRaw = require('ipld-raw') const multicodec = require('multicodec') const typical = require('typical') -const { fancyIterator } = require('./util') +const { extendIterator, fancyIterator } = require('./util') class IPLDResolver { constructor (userOptions) { @@ -195,42 +195,35 @@ class IPLDResolver { let options let formatImpl - const next = async () => { - // End iteration if there are no more nodes to put - if (nodes.length === 0) { - return { done: true } - } - - // Lazy load the options not when the iterator is initialized, but - // when we hit the first iteration. This way the constructor can be - // a synchronous function. - if (options === undefined) { - formatImpl = await this._getFormat(format) - const defaultOptions = { - hashAlg: formatImpl.defaultHashAlg, - cidVersion: 1, - onlyHash: false + const generator = async function * () { + for await (const node of nodes) { + // Lazy load the options not when the iterator is initialized, but + // when we hit the first iteration. This way the constructor can be + // a synchronous function. + if (options === undefined) { + formatImpl = await this._getFormat(format) + const defaultOptions = { + hashAlg: formatImpl.defaultHashAlg, + cidVersion: 1, + onlyHash: false + } + options = mergeOptions(defaultOptions, userOptions) } - options = mergeOptions(defaultOptions, userOptions) - } - const node = nodes.shift() - const cidOptions = { - version: options.cidVersion, - hashAlg: options.hashAlg, - onlyHash: options.onlyHash - } - const cid = await promisify(formatImpl.util.cid)(node, cidOptions) - if (!options.onlyHash) { - await this._store(cid, node) - } - return { - done: false, - value: cid + const cidOptions = { + version: options.cidVersion, + hashAlg: options.hashAlg, + onlyHash: options.onlyHash + } + const cid = await promisify(formatImpl.util.cid)(node, cidOptions) + if (!options.onlyHash) { + await this._store(cid, node) + } + yield cid } - } + }.bind(this) - return fancyIterator(next) + return extendIterator(generator()) } /** diff --git a/src/util.js b/src/util.js index 1e86375..db0be89 100644 --- a/src/util.js +++ b/src/util.js @@ -30,3 +30,10 @@ exports.fancyIterator = (next) => { iterator.all = () => exports.all(iterator) return iterator } + +exports.extendIterator = (iterator) => { + iterator.first = () => exports.first(iterator) + iterator.last = () => exports.last(iterator) + iterator.all = () => exports.all(iterator) + return iterator +} From 838aca819675813dca19bf854c620f881102b3da Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 21 Feb 2019 17:52:34 +0100 Subject: [PATCH 20/27] refactor: make get() a generator Instead of manually coding an iterator, use an ES2015 generator. --- src/index.js | 32 ++++++++------------------------ test/ipld-all.js | 28 ++++++++++------------------ 2 files changed, 18 insertions(+), 42 deletions(-) diff --git a/src/index.js b/src/index.js index a2af861..b86a45b 100644 --- a/src/index.js +++ b/src/index.js @@ -141,32 +141,16 @@ class IPLDResolver { throw new Error('`cids` must be an iterable of CIDs') } - let blocks - const next = async () => { - // End of iteration if there aren't any blocks left to return - if (cids.length === 0 || - (blocks !== undefined && blocks.length === 0) - ) { - return { done: true } - } - - // Lazy load block. - // Currently the BlockService return all nodes as an array. In the - // future this will also be an iterator - if (blocks === undefined) { - const cidsArray = Array.from(cids) - blocks = await promisify(this.bs.getMany.bind(this.bs))(cidsArray) - } - const block = blocks.shift() - const node = await this._deserialize(block) - - return { - done: false, - value: node + const generator = async function * () { + for await (const cid of cids) { + const block = await promisify(this.bs.get.bind(this.bs))(cid) + const format = await this._getFormat(block.cid.codec) + const node = await promisify(format.util.deserialize)(block.data) + yield node } - } + }.bind(this) - return fancyIterator(next) + return extendIterator(generator()) } /** diff --git a/test/ipld-all.js b/test/ipld-all.js index d21a5f0..8041e21 100644 --- a/test/ipld-all.js +++ b/test/ipld-all.js @@ -109,42 +109,34 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { describe('get', () => { it('should return nodes correctly', async () => { const result = resolver.get([cidCbor, cidPb]) - const node1 = await result.first() + const [node1, node2] = await result.all() expect(node1).to.eql(nodeCbor) - - const node2 = await result.first() expect(node2).to.eql(nodePb) }) it('should return nodes in input order', async () => { const result = resolver.get([cidPb, cidCbor]) - const node1 = await result.first() + const [node1, node2] = await result.all() expect(node1).to.eql(nodePb) - - const node2 = await result.first() expect(node2).to.eql(nodeCbor) }) it('should return error on invalid CID', async () => { const result = resolver.get([cidCbor, 'invalidcid']) - // TODO vmx 2018-12-11: This should really fail on the second node - // we get, as the first one is valid. This is only possible once - // the `getmany()` call of the BlockService takes and returns an - // iterator and not an array. - await expect(result.next()).to.be.rejectedWith( - 'Not a valid cid') + // First node is valid + await result.next() + // Second one is not + await expect(result.next()).to.be.rejectedWith('Not a valid cid') }) it('should return error on non-existent CID', async () => { const nonExistentCid = new CID( 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') const result = resolver.get([cidCbor, nonExistentCid]) - // TODO vmx 2018-12-11: This should really fail on the second node - // we get, as the first one is valid. This is only possible once - // the `getmany()` call of the BlockService takes and returns an - // iterator and not an array. - await expect(result.next()).to.be.rejectedWith( - 'Not Found') + // First node is valid + await result.next() + // Second one is not + await expect(result.next()).to.be.rejectedWith('Not Found') }) it('should return error on invalid input', () => { From d4a58d7279b25f5e3a2d6c7c2ef2d49bffdfd81f Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 21 Feb 2019 19:21:28 +0100 Subject: [PATCH 21/27] refactor: make resolve() a generator Instead of manually coding an iterator, use an ES2015 generator. --- src/index.js | 47 +++++++++++++++++++----------------------- test/ipld-all.js | 3 +-- test/ipld-bitcoin.js | 7 ++----- test/ipld-dag-cbor.js | 10 +++------ test/ipld-dag-pb.js | 7 ++----- test/ipld-eth-block.js | 7 ++----- test/ipld-eth.js | 3 +-- test/ipld-git.js | 17 +++++++-------- test/ipld-zcash.js | 7 ++----- 9 files changed, 42 insertions(+), 66 deletions(-) diff --git a/src/index.js b/src/index.js index b86a45b..599e477 100644 --- a/src/index.js +++ b/src/index.js @@ -93,40 +93,35 @@ class IPLDResolver { throw new Error('`path` argument must be a string') } - const next = async () => { + const generator = async function * () { // End iteration if there isn't a CID to follow anymore - if (cid === null) { - return { done: true } - } + while (cid !== null) { + const format = await this._getFormat(cid.codec) - const format = await this._getFormat(cid.codec) - - // get block - // use local resolver - // update path value - const block = await promisify(this.bs.get.bind(this.bs))(cid) - const result = await promisify(format.resolver.resolve)(block.data, path) - - // Prepare for the next iteration if there is a `remainderPath` - path = result.remainderPath - let value = result.value - // NOTE vmx 2018-11-29: Not all IPLD Formats return links as - // CIDs yet. Hence try to convert old style links to CIDs - if (Object.keys(value).length === 1 && '/' in value) { - value = new CID(value['/']) - } - cid = CID.isCID(value) ? value : null + // get block + // use local resolver + // update path value + const block = await promisify(this.bs.get.bind(this.bs))(cid) + const result = await promisify(format.resolver.resolve)(block.data, path) + + // Prepare for the next iteration if there is a `remainderPath` + path = result.remainderPath + let value = result.value + // NOTE vmx 2018-11-29: Not all IPLD Formats return links as + // CIDs yet. Hence try to convert old style links to CIDs + if (Object.keys(value).length === 1 && '/' in value) { + value = new CID(value['/']) + } + cid = CID.isCID(value) ? value : null - return { - done: false, - value: { + yield { remainderPath: path, value } } - } + }.bind(this) - return fancyIterator(next) + return extendIterator(generator()) } /** diff --git a/test/ipld-all.js b/test/ipld-all.js index 8041e21..a248525 100644 --- a/test/ipld-all.js +++ b/test/ipld-all.js @@ -71,12 +71,11 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { it('resolve through different formats', async () => { const result = resolver.resolve(cidCbor, 'pb/Data') + const [node1, node2] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('Data') expect(node1.value).to.eql(cidPb) - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value).to.eql(Buffer.from('I am inside a Protobuf')) }) diff --git a/test/ipld-bitcoin.js b/test/ipld-bitcoin.js index f7bd238..8840c83 100644 --- a/test/ipld-bitcoin.js +++ b/test/ipld-bitcoin.js @@ -123,28 +123,25 @@ module.exports = (repo) => { it('resolves value within nested scope (1 level)', async () => { const result = resolver.resolve(cid2, 'parent/version') + const [node1, node2] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('version') expect(node1.value).to.eql(cid1) - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value).to.eql(1) }) it('resolves value within nested scope (2 levels)', async () => { const result = resolver.resolve(cid3, 'parent/parent/version') + const [node1, node2, node3] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('parent/version') expect(node1.value).to.eql(cid2) - const node2 = await result.first() expect(node2.remainderPath).to.eql('version') expect(node2.value).to.eql(cid1) - const node3 = await result.first() expect(node3.remainderPath).to.eql('') expect(node3.value).to.eql(1) }) diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index b9f8cb0..ba5b0f9 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -110,40 +110,36 @@ module.exports = (repo) => { it('resolves value within nested scope (0 level)', async () => { const result = resolver.resolve(cid2, 'one') + const [node1, node2] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('') expect(node1.value).to.eql(cid1) - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value).to.eql({ someData: 'I am 1' }) }) it('resolves value within nested scope (1 level)', async () => { const result = resolver.resolve(cid2, 'one/someData') + const [node1, node2] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('someData') expect(node1.value).to.eql(cid1) - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value).to.eql('I am 1') }) it('resolves value within nested scope (2 levels)', async () => { const result = resolver.resolve(cid3, 'two/one/someData') + const [node1, node2, node3] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('one/someData') expect(node1.value).to.eql(cid2) - const node2 = await result.first() expect(node2.remainderPath).to.eql('someData') expect(node2.value).to.eql(cid1) - const node3 = await result.first() expect(node3.remainderPath).to.eql('') expect(node3.value).to.eql('I am 1') }) diff --git a/test/ipld-dag-pb.js b/test/ipld-dag-pb.js index ed7a5fc..213f1cc 100644 --- a/test/ipld-dag-pb.js +++ b/test/ipld-dag-pb.js @@ -139,28 +139,25 @@ module.exports = (repo) => { it('resolves a value within nested scope (1 level)', async () => { const result = resolver.resolve(cid2, 'Links/0/Hash/Data') + const [node1, node2] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('Data') expect(node1.value).to.eql(cid1.toV0()) - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value).to.eql(Buffer.from('I am 1')) }) it('resolves value within nested scope (2 levels)', async () => { const result = resolver.resolve(cid3, 'Links/1/Hash/Links/0/Hash/Data') + const [node1, node2, node3] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('Links/0/Hash/Data') expect(node1.value).to.eql(cid2.toV0()) - const node2 = await result.first() expect(node2.remainderPath).to.eql('Data') expect(node2.value).to.eql(cid1.toV0()) - const node3 = await result.first() expect(node3.remainderPath).to.eql('') expect(node3.value).to.eql(Buffer.from('I am 1')) }) diff --git a/test/ipld-eth-block.js b/test/ipld-eth-block.js index ddf9ab7..63e414b 100644 --- a/test/ipld-eth-block.js +++ b/test/ipld-eth-block.js @@ -82,28 +82,25 @@ module.exports = (repo) => { it('resolves value within nested scope (1 level)', async () => { const result = resolver.resolve(cid2, 'parent/number') + const [node1, node2] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('number') expect(node1.value).to.eql(cid1) - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value.toString('hex')).to.eql('01') }) it('resolves value within nested scope (2 levels)', async () => { const result = resolver.resolve(cid3, 'parent/parent/number') + const [node1, node2, node3] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('parent/number') expect(node1.value).to.eql(cid2) - const node2 = await result.first() expect(node2.remainderPath).to.eql('number') expect(node2.value).to.eql(cid1) - const node3 = await result.first() expect(node3.remainderPath).to.eql('') expect(node3.value.toString('hex')).to.eql('01') }) diff --git a/test/ipld-eth.js b/test/ipld-eth.js index ed6a200..a727408 100644 --- a/test/ipld-eth.js +++ b/test/ipld-eth.js @@ -90,11 +90,10 @@ module.exports = (repo) => { it('block-to-block', async () => { const child = await ethObjs.child const result = resolver.resolve(child.cid, 'parent') + const [node1, node2] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('') - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value.number.toString('hex')).to.eql('302516') }) diff --git a/test/ipld-git.js b/test/ipld-git.js index 15549c5..e535b1e 100644 --- a/test/ipld-git.js +++ b/test/ipld-git.js @@ -174,41 +174,39 @@ module.exports = (repo) => { it('resolves value within nested node scope (commit/tree)', async () => { const result = resolver.resolve(commitCid, 'tree/somefile/mode') + const [node1, node2] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('somefile/mode') expect(node1.value).to.eql(treeCid) - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value).to.eql('100644') }) it('resolves value within nested node scope (commit/tree/blob)', async () => { const result = resolver.resolve(commitCid, 'tree/somefile/hash') + const [node1, node2, node3] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('somefile/hash') expect(node1.value).to.eql(treeCid) - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value).to.eql(blobCid) - const node3 = await result.first() expect(node3.remainderPath).to.eql('') expect(node3.value).to.eql(blobNode) }) it('resolves value within nested node scope (commit/commit/tree/blob)', async () => { const result = resolver.resolve(commit2Cid, 'parents/0/tree/somefile/hash') + const nodes = await result.all() - const node1 = await result.first() + const node1 = nodes.shift() expect(node1.remainderPath).to.eql('tree/somefile/hash') expect(node1.value).to.eql(commitCid) // The nodes in between were already tested by some other test - const last = await result.last() + const last = await nodes.pop() expect(last.remainderPath).to.eql('') expect(last.value).to.eql(blobNode) }) @@ -216,13 +214,14 @@ module.exports = (repo) => { it('resolves value within nested node scope (tag/commit/commit/tree/blob)', async () => { const result = resolver.resolve(tagCid, 'object/parents/0/tree/somefile/hash') + const nodes = await result.all() - const node1 = await result.first() + const node1 = nodes.shift() expect(node1.remainderPath).to.eql('parents/0/tree/somefile/hash') expect(node1.value).to.eql(commit2Cid) // The nodes in between were already tested by some other test - const last = await result.last() + const last = nodes.pop() expect(last.remainderPath).to.eql('') expect(last.value).to.eql(blobNode) }) diff --git a/test/ipld-zcash.js b/test/ipld-zcash.js index a966e32..b346258 100644 --- a/test/ipld-zcash.js +++ b/test/ipld-zcash.js @@ -128,28 +128,25 @@ module.exports = (repo) => { it('resolves value within nested scope (1 level)', async () => { const result = resolver.resolve(cid2, 'parent/version') + const [node1, node2] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('version') expect(node1.value).to.eql(cid1) - const node2 = await result.first() expect(node2.remainderPath).to.eql('') expect(node2.value).to.eql(1) }) it('resolves value within nested scope (2 levels)', async () => { const result = resolver.resolve(cid3, 'parent/parent/version') + const [node1, node2, node3] = await result.all() - const node1 = await result.first() expect(node1.remainderPath).to.eql('parent/version') expect(node1.value).to.eql(cid2) - const node2 = await result.first() expect(node2.remainderPath).to.eql('version') expect(node2.value).to.eql(cid1) - const node3 = await result.first() expect(node3.remainderPath).to.eql('') expect(node3.value).to.eql(1) }) From d0bf6dea52ea2b6ad5661a93e268c6dfd4bb9658 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 21 Feb 2019 19:26:35 +0100 Subject: [PATCH 22/27] refactor: make remove() a generator Instead of manually coding an iterator, use an ES2015 generator. --- src/index.js | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/index.js b/src/index.js index 599e477..bbc8b13 100644 --- a/src/index.js +++ b/src/index.js @@ -220,22 +220,14 @@ class IPLDResolver { throw new Error('`cids` must be an iterable of CIDs') } - const next = async () => { - // End iteration if there are no more nodes to remove - if (cids.length === 0) { - return { done: true } - } - - const cid = cids.shift() - await promisify(this.bs.delete.bind(this.bs))(cid) - - return { - done: false, - value: cid + const generator = async function * () { + for await (const cid of cids) { + await promisify(this.bs.delete.bind(this.bs))(cid) + yield cid } - } + }.bind(this) - return fancyIterator(next) + return extendIterator(generator()) } /** From 381e8dc89d1b7e178287e4ac900eb01acc4e09c2 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 21 Feb 2019 19:37:47 +0100 Subject: [PATCH 23/27] refactor: make tree() a generator Instead of manually coding an iterator, use an ES2015 generator. --- src/index.js | 90 ++++++++++++++++++++++++---------------------------- src/util.js | 9 ------ 2 files changed, 42 insertions(+), 57 deletions(-) diff --git a/src/index.js b/src/index.js index bbc8b13..e92748c 100644 --- a/src/index.js +++ b/src/index.js @@ -10,7 +10,7 @@ const ipldDagPb = require('ipld-dag-pb') const ipldRaw = require('ipld-raw') const multicodec = require('multicodec') const typical = require('typical') -const { extendIterator, fancyIterator } = require('./util') +const { extendIterator } = require('./util') class IPLDResolver { constructor (userOptions) { @@ -266,63 +266,57 @@ class IPLDResolver { } } - // The list of paths that will get returned - let treePaths = [] - // The current block, needed to call `isLink()` on every interation - let block - // The list of items we want to follow recursively. The items are - // an object consisting of the CID and the currently already resolved - // path - const queue = [{ cid, basePath: '' }] - // The path that was already traversed - let basePath - - const next = async () => { + const generator = async function * () { + // The list of paths that will get returned + const treePaths = [] + // The current block, needed to call `isLink()` on every interation + let block + // The list of items we want to follow recursively. The items are + // an object consisting of the CID and the currently already resolved + // path + const queue = [{ cid, basePath: '' }] + // The path that was already traversed + let basePath + // End of iteration if there aren't any paths left to return or // if we don't want to traverse recursively and have already // returne the first level - if (treePaths.length === 0 && queue.length === 0) { - return { done: true } - } - - // There aren't any paths left, get them from the given CID - if (treePaths.length === 0 && queue.length > 0) { - ({ cid, basePath } = queue.shift()) - const format = await this._getFormat(cid.codec) - block = await promisify(this.bs.get.bind(this.bs))(cid) - - const paths = await promisify(format.resolver.tree)(block.data) - treePaths.push(...paths) - } + while (treePaths.length > 0 || queue.length > 0) { + // There aren't any paths left, get them from the given CID + if (treePaths.length === 0 && queue.length > 0) { + ({ cid, basePath } = queue.shift()) + const format = await this._getFormat(cid.codec) + block = await promisify(this.bs.get.bind(this.bs))(cid) + + const paths = await promisify(format.resolver.tree)(block.data) + treePaths.push(...paths) + } - const treePath = treePaths.shift() - let fullPath = basePath + treePath + const treePath = treePaths.shift() + let fullPath = basePath + treePath - // Only follow links if recursion is intended - if (options.recursive) { - cid = await maybeRecurse(block, treePath) - if (cid !== null) { - queue.push({ cid, basePath: fullPath + '/' }) + // Only follow links if recursion is intended + if (options.recursive) { + cid = await maybeRecurse(block, treePath) + if (cid !== null) { + queue.push({ cid, basePath: fullPath + '/' }) + } } - } - // Return it if it matches the given offset path, but is not the - // offset path itself - if (fullPath.startsWith(offsetPath) && - fullPath.length > offsetPath.length) { - if (offsetPath.length > 0) { - fullPath = fullPath.slice(offsetPath.length + 1) - } - return { - done: false, - value: fullPath + // Return it if it matches the given offset path, but is not the + // offset path itself + if (fullPath.startsWith(offsetPath) && + fullPath.length > offsetPath.length) { + if (offsetPath.length > 0) { + fullPath = fullPath.slice(offsetPath.length + 1) + } + + yield fullPath } - } else { // Else move on to the next iteration before returning - return next() } - } + }.bind(this) - return fancyIterator(next) + return extendIterator(generator()) } /* */ diff --git a/src/util.js b/src/util.js index db0be89..b5a1644 100644 --- a/src/util.js +++ b/src/util.js @@ -22,15 +22,6 @@ exports.all = async (iterator) => { return values } -exports.fancyIterator = (next) => { - const iterator = { next } - iterator[Symbol.asyncIterator] = function () { return this } - iterator.first = () => exports.first(iterator) - iterator.last = () => exports.last(iterator) - iterator.all = () => exports.all(iterator) - return iterator -} - exports.extendIterator = (iterator) => { iterator.first = () => exports.first(iterator) iterator.last = () => exports.last(iterator) From a72f13d9268ba1ff3550641dd57b95f1e238aff6 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 28 Feb 2019 21:59:41 +0100 Subject: [PATCH 24/27] fix: use promisify-es6 instead of Nodes.js' promisify js-ipfs is using the promisify-es6 module, so it makes sense that we use it as well. --- package.json | 1 + src/index.js | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index dda3656..285056b 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "ipld-raw": "^2.0.1", "merge-options": "^1.0.1", "multicodec": "~0.5.0", + "promisify-es6": "^1.0.3", "typical": "^3.0.0" }, "contributors": [ diff --git a/src/index.js b/src/index.js index e92748c..0184a6e 100644 --- a/src/index.js +++ b/src/index.js @@ -1,7 +1,5 @@ 'use strict' -const promisify = require('util').promisify - const Block = require('ipfs-block') const CID = require('cids') const mergeOptions = require('merge-options') @@ -9,6 +7,7 @@ const ipldDagCbor = require('ipld-dag-cbor') const ipldDagPb = require('ipld-dag-pb') const ipldRaw = require('ipld-raw') const multicodec = require('multicodec') +const promisify = require('promisify-es6') const typical = require('typical') const { extendIterator } = require('./util') From af2dfd76b6c5f5597d6b880a374d100c92144f71 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 28 Feb 2019 22:00:35 +0100 Subject: [PATCH 25/27] fix: don't throw if it's not a proper old-style link We are still supporting old style links, i.e. JSON where `/` as key signals a link. Though the JSON could could also contain such a key without having a CID as value. Instead of throwing an error, treat it as not being a link. --- src/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 0184a6e..0f23115 100644 --- a/src/index.js +++ b/src/index.js @@ -109,7 +109,11 @@ class IPLDResolver { // NOTE vmx 2018-11-29: Not all IPLD Formats return links as // CIDs yet. Hence try to convert old style links to CIDs if (Object.keys(value).length === 1 && '/' in value) { - value = new CID(value['/']) + try { + value = new CID(value['/']) + } catch (_error) { + value = null + } } cid = CID.isCID(value) ? value : null From 185814a3b3dca634f3d7e8123ee5206a7a1b7197 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Fri, 1 Mar 2019 11:15:48 +0100 Subject: [PATCH 26/27] fix: use a version of typical where async iterators are supported More recent versions of typical detect async itertators as an iterable. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 285056b..25b075d 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "merge-options": "^1.0.1", "multicodec": "~0.5.0", "promisify-es6": "^1.0.3", - "typical": "^3.0.0" + "typical": "^3.0.2" }, "contributors": [ "Alan Shaw ", From 4666a54e921dfcf020e83d08f61137f8d9ab3290 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Mon, 18 Mar 2019 16:43:01 +0100 Subject: [PATCH 27/27] feat: add single item functions BREAKING CHANGE: put/get/remove functions are renamed This commit introduces single item functions which are called `put()`/`get()`,`remove()`. In order to put, get or remove multiple items you need to call `putMany()`,`getMany()`/`removeMany()` now. --- src/index.js | 96 ++++++++++++++++++++++++++++++++---------- test/basics.js | 20 +++++++-- test/format-support.js | 3 +- test/ipld-all.js | 19 ++++----- test/ipld-bitcoin.js | 27 ++++-------- test/ipld-dag-cbor.js | 27 ++++-------- test/ipld-dag-pb.js | 27 ++++-------- test/ipld-eth-block.js | 27 ++++-------- test/ipld-eth.js | 3 +- test/ipld-git.js | 27 ++++-------- test/ipld-zcash.js | 27 ++++-------- 11 files changed, 156 insertions(+), 147 deletions(-) diff --git a/src/index.js b/src/index.js index 0f23115..09273df 100644 --- a/src/index.js +++ b/src/index.js @@ -127,13 +127,27 @@ class IPLDResolver { return extendIterator(generator()) } + /** + * Get a node by CID. + * + * @param {CID} cid - The CID of the IPLD Node that should be retrieved. + * @returns {Promise.} - Returns a Promise with the IPLD Node that correspond to the given `cid`. + */ + async get (cid) { + const block = await promisify(this.bs.get.bind(this.bs))(cid) + const format = await this._getFormat(block.cid.codec) + const node = await promisify(format.util.deserialize)(block.data) + + return node + } + /** * Get multiple nodes back from an array of CIDs. * * @param {Iterable.} cids - The CIDs of the IPLD Nodes that should be retrieved. * @returns {Iterable.>} - Returns an async iterator with the IPLD Nodes that correspond to the given `cids`. */ - get (cids) { + getMany (cids) { if (!typical.isIterable(cids) || typical.isString(cids) || Buffer.isBuffer(cids)) { throw new Error('`cids` must be an iterable of CIDs') @@ -141,16 +155,54 @@ class IPLDResolver { const generator = async function * () { for await (const cid of cids) { - const block = await promisify(this.bs.get.bind(this.bs))(cid) - const format = await this._getFormat(block.cid.codec) - const node = await promisify(format.util.deserialize)(block.data) - yield node + yield this.get(cid) } }.bind(this) return extendIterator(generator()) } + /** + * Stores the given IPLD Node of a recognized IPLD Format. + * + * @param {Object} node - The deserialized IPLD node that should be inserted. + * @param {number} format - The multicodec of the format that IPLD Node should be encoded in. + * @param {Object} [userOptions] - Options is an object with the following properties. + * @param {number} [userOtions.hashAlg=hash algorithm of the given multicodec] - The hashing algorithm that is used to calculate the CID. + * @param {number} [userOptions.cidVersion=1] - The CID version to use. + * @param {boolean} [userOptions.onlyHash=false] - If true the serialized form of the IPLD Node will not be passed to the underlying block store. + * @returns {Promise.} - Returns the CID of the serialized IPLD Nodes. + */ + async put (node, format, userOptions) { + if (format === undefined) { + throw new Error('`put` requires a format') + } + if (typeof format !== 'number') { + throw new Error('`format` parameter must be number (multicodec)') + } + + const formatImpl = await this._getFormat(format) + const defaultOptions = { + hashAlg: formatImpl.defaultHashAlg, + cidVersion: 1, + onlyHash: false + } + const options = mergeOptions(defaultOptions, userOptions) + + const cidOptions = { + version: options.cidVersion, + hashAlg: options.hashAlg, + onlyHash: options.onlyHash + } + const cid = await promisify(formatImpl.util.cid)(node, cidOptions) + + if (!options.onlyHash) { + await this._store(cid, node) + } + + return cid + } + /** * Stores the given IPLD Nodes of a recognized IPLD Format. * @@ -158,11 +210,11 @@ class IPLDResolver { * @param {number} format - The multicodec of the format that IPLD Node should be encoded in. * @param {Object} [userOptions] - Options are applied to any of the `nodes` and is an object with the following properties. * @param {number} [userOtions.hashAlg=hash algorithm of the given multicodec] - The hashing algorithm that is used to calculate the CID. - * @param {number} [userOptions.cidVersion=1]`- The CID version to use. + * @param {number} [userOptions.cidVersion=1] - The CID version to use. * @param {boolean} [userOptions.onlyHash=false] - If true the serialized form of the IPLD Node will not be passed to the underlying block store. * @returns {Iterable.>} - Returns an async iterator with the CIDs of the serialized IPLD Nodes. */ - put (nodes, format, userOptions) { + putMany (nodes, format, userOptions) { if (!typical.isIterable(nodes) || typical.isString(nodes) || Buffer.isBuffer(nodes)) { throw new Error('`nodes` must be an iterable') @@ -192,32 +244,33 @@ class IPLDResolver { options = mergeOptions(defaultOptions, userOptions) } - const cidOptions = { - version: options.cidVersion, - hashAlg: options.hashAlg, - onlyHash: options.onlyHash - } - const cid = await promisify(formatImpl.util.cid)(node, cidOptions) - if (!options.onlyHash) { - await this._store(cid, node) - } - yield cid + yield this.put(node, format, options) } }.bind(this) return extendIterator(generator()) } + /** + * Remove an IPLD Node by the given CID. + * + * @param {CID} cid - The CID of the IPLD Node that should be removed. + * @return {Promise.} The CID of the removed IPLD Node. + */ + async remove (cid) { + return promisify(this.bs.delete.bind(this.bs))(cid) + } + /** * Remove IPLD Nodes by the given CIDs. * * Throws an error if any of the Blocks can’t be removed. This operation is * *not* atomic, some Blocks might have already been removed. * - * @param {Iterable.} cids - The CIDs of the IPLD Nodes that should be removed - * @return {void} + * @param {Iterable.} cids - The CIDs of the IPLD Nodes that should be removed. + * @return {Iterable.>} Returns an async iterator with the CIDs of the removed IPLD Nodes. */ - remove (cids) { + removeMany (cids) { if (!typical.isIterable(cids) || typical.isString(cids) || Buffer.isBuffer(cids)) { throw new Error('`cids` must be an iterable of CIDs') @@ -225,8 +278,7 @@ class IPLDResolver { const generator = async function * () { for await (const cid of cids) { - await promisify(this.bs.delete.bind(this.bs))(cid) - yield cid + yield this.remove(cid) } }.bind(this) diff --git a/test/basics.js b/test/basics.js index 4034a1c..622a1d0 100644 --- a/test/basics.js +++ b/test/basics.js @@ -53,15 +53,29 @@ module.exports = (repo) => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) // choosing a format that is not supported - const result = r.put([null], multicodec.BLAKE2B_8) + await expect(r.put(null, multicodec.BLAKE2B_8)).to.be.rejectedWith( + 'No resolver found for codec "blake2b-8"') + }) + + it('put - errors if no format is provided', async () => { + const bs = new BlockService(repo) + const r = new IPLDResolver({ blockService: bs }) + await expect(r.put(null)).to.be.rejectedWith('`put` requires a format') + }) + + it('putMany - errors on unknown resolver', async () => { + const bs = new BlockService(repo) + const r = new IPLDResolver({ blockService: bs }) + // choosing a format that is not supported + const result = r.putMany([null], multicodec.BLAKE2B_8) await expect(result.next()).to.be.rejectedWith( 'No resolver found for codec "blake2b-8"') }) - it('put - errors if no format is provided', () => { + it('putMany - errors if no format is provided', () => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) - expect(() => r.put([null])).to.be.throw('`put` requires a format') + expect(() => r.putMany([null])).to.be.throw('`put` requires a format') }) it('tree - errors on unknown resolver', async () => { diff --git a/test/format-support.js b/test/format-support.js index dc362d3..2171fc1 100644 --- a/test/format-support.js +++ b/test/format-support.js @@ -23,8 +23,7 @@ module.exports = (repo) => { data = { now: Date.now() } - const result = resolver.put([data], multicodec.DAG_CBOR) - cid = await result.last() + cid = await resolver.put(data, multicodec.DAG_CBOR) }) describe('Dynamic format loading', () => { diff --git a/test/ipld-all.js b/test/ipld-all.js index a248525..5d43ebb 100644 --- a/test/ipld-all.js +++ b/test/ipld-all.js @@ -58,9 +58,9 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { { node: nodePb, format: multicodec.DAG_PB, cidVersion: 0 }, { node: nodeCbor, format: multicodec.DAG_CBOR, cidVersion: 1 } ], (nac, cb) => { - resolver.put([nac.node], nac.format, { + resolver.put(nac.node, nac.format, { cidVersion: nac.cidVersion - }).first().then( + }).then( () => cb(null), (error) => cb(error) ) @@ -84,12 +84,11 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { waterfall([ (cb) => dagPB.DAGNode.create(Buffer.from('Some data here'), cb), (node, cb) => { - const result = resolver.put([node], multicodec.DAG_PB, { + resolver.put(node, multicodec.DAG_PB, { onlyHash: true, cidVersion: 1, hashAlg: multicodec.SHA2_256 - }) - result.first().then( + }).then( (cid) => cb(null, cid), (error) => cb(error) ) @@ -107,21 +106,21 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { describe('get', () => { it('should return nodes correctly', async () => { - const result = resolver.get([cidCbor, cidPb]) + const result = resolver.getMany([cidCbor, cidPb]) const [node1, node2] = await result.all() expect(node1).to.eql(nodeCbor) expect(node2).to.eql(nodePb) }) it('should return nodes in input order', async () => { - const result = resolver.get([cidPb, cidCbor]) + const result = resolver.getMany([cidPb, cidCbor]) const [node1, node2] = await result.all() expect(node1).to.eql(nodePb) expect(node2).to.eql(nodeCbor) }) it('should return error on invalid CID', async () => { - const result = resolver.get([cidCbor, 'invalidcid']) + const result = resolver.getMany([cidCbor, 'invalidcid']) // First node is valid await result.next() // Second one is not @@ -131,7 +130,7 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { it('should return error on non-existent CID', async () => { const nonExistentCid = new CID( 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - const result = resolver.get([cidCbor, nonExistentCid]) + const result = resolver.getMany([cidCbor, nonExistentCid]) // First node is valid await result.next() // Second one is not @@ -139,7 +138,7 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { }) it('should return error on invalid input', () => { - expect(() => resolver.get('astring')).to.throw( + expect(() => resolver.getMany('astring')).to.throw( '`cids` must be an iterable of CIDs') }) }) diff --git a/test/ipld-bitcoin.js b/test/ipld-bitcoin.js index 8840c83..42572c0 100644 --- a/test/ipld-bitcoin.js +++ b/test/ipld-bitcoin.js @@ -84,7 +84,7 @@ module.exports = (repo) => { async function store () { const nodes = [node1, node2, node3] - const result = resolver.put(nodes, multicodec.BITCOIN_BLOCK) + const result = resolver.putMany(nodes, multicodec.BITCOIN_BLOCK) ;[cid1, cid2, cid3] = await result.all() done() @@ -93,8 +93,7 @@ module.exports = (repo) => { describe('public api', () => { it('resolver.put with format', async () => { - const result = resolver.put([node1], multicodec.BITCOIN_BLOCK) - const cid = await result.first() + const cid = await resolver.put(node1, multicodec.BITCOIN_BLOCK) expect(cid.version).to.equal(1) expect(cid.codec).to.equal('bitcoin-block') expect(cid.multihash).to.exist() @@ -103,10 +102,9 @@ module.exports = (repo) => { }) it('resolver.put with format + hashAlg', async () => { - const result = resolver.put([node1], multicodec.BITCOIN_BLOCK, { + const cid = await resolver.put(node1, multicodec.BITCOIN_BLOCK, { hashAlg: multicodec.SHA3_512 }) - const cid = await result.first() expect(cid.version).to.equal(1) expect(cid.codec).to.equal('bitcoin-block') expect(cid.multihash).to.exist() @@ -147,28 +145,21 @@ module.exports = (repo) => { }) it('resolver.get round-trip', async () => { - const resultPut = resolver.put([node1], multicodec.BITCOIN_BLOCK) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const node = await resultGet.first() + const cid = await resolver.put(node1, multicodec.BITCOIN_BLOCK) + const node = await resolver.get(cid) expect(node).to.deep.equal(node1) }) it('resolver.remove', async () => { - const resultPut = resolver.put([node1], multicodec.BITCOIN_BLOCK) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const sameAsNode1 = await resultGet.first() + const cid = await resolver.put(node1, multicodec.BITCOIN_BLOCK) + const sameAsNode1 = await resolver.get(cid) expect(sameAsNode1).to.deep.equal(node1) return remove() async function remove () { - const resultRemove = resolver.remove([cid]) - // The items are deleted through iteration - await resultRemove.last() + await resolver.remove(cid) // Verify that the item got really deleted - const resultGet = resolver.get([cid]) - await expect(resultGet.next()).to.eventually.be.rejected() + await expect(resolver.get(cid)).to.eventually.be.rejected() } }) }) diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index ba5b0f9..4ace848 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -70,7 +70,7 @@ module.exports = (repo) => { async function store () { const nodes = [node1, node2, node3] - const result = resolver.put(nodes, multicodec.DAG_CBOR) + const result = resolver.putMany(nodes, multicodec.DAG_CBOR) ;[cid1, cid2, cid3] = await result.all() done() @@ -79,8 +79,7 @@ module.exports = (repo) => { describe('public api', () => { it('resolver.put with format', async () => { - const result = resolver.put([node1], multicodec.DAG_CBOR) - const cid = await result.first() + const cid = await resolver.put(node1, multicodec.DAG_CBOR) expect(cid.version).to.equal(1) expect(cid.codec).to.equal('dag-cbor') expect(cid.multihash).to.exist() @@ -89,10 +88,9 @@ module.exports = (repo) => { }) it('resolver.put with format + hashAlg', async () => { - const result = resolver.put([node1], multicodec.DAG_CBOR, { + const cid = await resolver.put(node1, multicodec.DAG_CBOR, { hashAlg: multicodec.SHA3_512 }) - const cid = await result.first() expect(cid).to.exist() expect(cid.version).to.equal(1) expect(cid.codec).to.equal('dag-cbor') @@ -151,10 +149,8 @@ module.exports = (repo) => { }) it('resolver.get round-trip', async () => { - const resultPut = resolver.put([node1], multicodec.DAG_CBOR) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const node = await resultGet.first() + const cid = await resolver.put(node1, multicodec.DAG_CBOR) + const node = await resolver.get(cid) expect(node).to.deep.equal(node1) }) @@ -205,20 +201,15 @@ module.exports = (repo) => { }) it('resolver.remove', async () => { - const resultPut = resolver.put([node1], multicodec.DAG_CBOR) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const sameAsNode1 = await resultGet.first() + const cid = await resolver.put(node1, multicodec.DAG_CBOR) + const sameAsNode1 = await resolver.get(cid) expect(sameAsNode1).to.deep.equal(node1) return remove() async function remove () { - const resultRemove = resolver.remove([cid]) - // The items are deleted through iteration - await resultRemove.last() + await resolver.remove(cid) // Verify that the item got really deleted - const resultGet = resolver.get([cid]) - await expect(resultGet.next()).to.eventually.be.rejected() + await expect(resolver.get(cid)).to.eventually.be.rejected() } }) }) diff --git a/test/ipld-dag-pb.js b/test/ipld-dag-pb.js index 213f1cc..331218f 100644 --- a/test/ipld-dag-pb.js +++ b/test/ipld-dag-pb.js @@ -100,7 +100,7 @@ module.exports = (repo) => { async function store () { const nodes = [node1, node2, node3] - const result = resolver.put(nodes, multicodec.DAG_PB) + const result = resolver.putMany(nodes, multicodec.DAG_PB) ;[cid1, cid2, cid3] = await result.all() done() @@ -109,8 +109,7 @@ module.exports = (repo) => { describe('public api', () => { it('resolver.put with format', async () => { - const result = resolver.put([node1], multicodec.DAG_PB) - const cid = await result.first() + const cid = await resolver.put(node1, multicodec.DAG_PB) expect(cid.version).to.equal(1) expect(cid.codec).to.equal('dag-pb') expect(cid.multihash).to.exist() @@ -119,10 +118,9 @@ module.exports = (repo) => { }) it('resolver.put with format + hashAlg', async () => { - const result = resolver.put([node1], multicodec.DAG_PB, { + const cid = await resolver.put(node1, multicodec.DAG_PB, { hashAlg: multicodec.SHA3_512 }) - const cid = await result.first() expect(cid.version).to.equal(1) expect(cid.codec).to.equal('dag-pb') expect(cid.multihash).to.exist() @@ -163,10 +161,8 @@ module.exports = (repo) => { }) it('resolver.get round-trip', async () => { - const resultPut = resolver.put([node1], multicodec.DAG_PB) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const node = await resultGet.first() + const cid = await resolver.put(node1, multicodec.DAG_PB) + const node = await resolver.get(cid) // `size` is lazy, without a call to it a deep equal check would fail const _ = node.size // eslint-disable-line no-unused-vars expect(node).to.deep.equal(node1) @@ -187,22 +183,17 @@ module.exports = (repo) => { }) }) const node = await createNode - const resultPut = resolver.put([node], multicodec.DAG_PB) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const sameAsNode = await resultGet.first() + const cid = await resolver.put(node, multicodec.DAG_PB) + const sameAsNode = await resolver.get(cid) // `size` is lazy, without a call to it a deep equal check would fail const _ = sameAsNode.size // eslint-disable-line no-unused-vars expect(sameAsNode.data).to.deep.equal(node.data) return remove() async function remove () { - const resultRemove = resolver.remove([cid]) - // The items are deleted through iteration - await resultRemove.last() + await resolver.remove(cid) // Verify that the item got really deleted - const resultGet = resolver.get([cid]) - await expect(resultGet.next()).to.eventually.be.rejected() + await expect(resolver.get(cid)).to.eventually.be.rejected() } }) }) diff --git a/test/ipld-eth-block.js b/test/ipld-eth-block.js index 63e414b..e77eec8 100644 --- a/test/ipld-eth-block.js +++ b/test/ipld-eth-block.js @@ -46,14 +46,13 @@ module.exports = (repo) => { }) const nodes = [node1, node2, node3] - const result = resolver.put(nodes, multicodec.ETH_BLOCK) + const result = resolver.putMany(nodes, multicodec.ETH_BLOCK) ;[cid1, cid2, cid3] = await result.all() }) describe('public api', () => { it('resolver.put with format', async () => { - const result = resolver.put([node1], multicodec.ETH_BLOCK) - const cid = await result.first() + const cid = await resolver.put(node1, multicodec.ETH_BLOCK) expect(cid.version).to.equal(1) expect(cid.codec).to.equal('eth-block') expect(cid.multihash).to.exist() @@ -62,10 +61,9 @@ module.exports = (repo) => { }) it('resolver.put with format + hashAlg', async () => { - const result = resolver.put([node1], multicodec.ETH_BLOCK, { + const cid = await resolver.put(node1, multicodec.ETH_BLOCK, { hashAlg: multicodec.KECCAK_512 }) - const cid = await result.first() expect(cid.version).to.equal(1) expect(cid.codec).to.equal('eth-block') expect(cid.multihash).to.exist() @@ -106,29 +104,22 @@ module.exports = (repo) => { }) it('resolver.get round-trip', async () => { - const resultPut = resolver.put([node1], multicodec.ETH_BLOCK) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const node = await resultGet.first() + const cid = await resolver.put(node1, multicodec.ETH_BLOCK) + const node = await resolver.get(cid) // TODO vmx 2018-12-12: Find out why the full nodes not deep equal expect(node.raw).to.deep.equal(node1.raw) }) it('resolver.remove', async () => { - const resultPut = resolver.put([node1], multicodec.ETH_BLOCK) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const sameAsNode1 = await resultGet.first() + const cid = await resolver.put(node1, multicodec.ETH_BLOCK) + const sameAsNode1 = await resolver.get(cid) expect(sameAsNode1.raw).to.deep.equal(node1.raw) return remove() async function remove () { - const resultRemove = resolver.remove([cid]) - // The items are deleted through iteration - await resultRemove.last() + await resolver.remove(cid) // Verify that the item got really deleted - const resultGet = resolver.get([cid]) - await expect(resultGet.next()).to.eventually.be.rejected() + await expect(resolver.get(cid)).to.eventually.be.rejected() } }) }) diff --git a/test/ipld-eth.js b/test/ipld-eth.js index a727408..27af1b7 100644 --- a/test/ipld-eth.js +++ b/test/ipld-eth.js @@ -75,8 +75,7 @@ module.exports = (repo) => { default: throw new Error('Unknown type!') } - const result = resolver.put([node], type) - const cid = await result.first() + const cid = await resolver.put(node, type) return { raw: rawData, diff --git a/test/ipld-git.js b/test/ipld-git.js index e535b1e..86636f7 100644 --- a/test/ipld-git.js +++ b/test/ipld-git.js @@ -135,7 +135,7 @@ module.exports = (repo) => { async function store () { const nodes = [blobNode, treeNode, commitNode, commit2Node, tagNode] - const result = resolver.put(nodes, multicodec.GIT_RAW) + const result = resolver.putMany(nodes, multicodec.GIT_RAW) ;[blobCid, treeCid, commitCid, commit2Cid, tagCid] = await result.all() done() @@ -144,8 +144,7 @@ module.exports = (repo) => { describe('public api', () => { it('resolver.put with format', async () => { - const result = resolver.put([blobNode], multicodec.GIT_RAW) - const cid = await result.first() + const cid = await resolver.put(blobNode, multicodec.GIT_RAW) expect(cid.version).to.equal(1) expect(cid.codec).to.equal('git-raw') expect(cid.multihash).to.exist() @@ -154,10 +153,9 @@ module.exports = (repo) => { }) it('resolver.put with format + hashAlg', async () => { - const result = resolver.put([blobNode], multicodec.GIT_RAW, { + const cid = await resolver.put(blobNode, multicodec.GIT_RAW, { hashAlg: multicodec.SHA3_512 }) - const cid = await result.first() expect(cid.version).to.equal(1) expect(cid.codec).to.equal('git-raw') expect(cid.multihash).to.exist() @@ -227,28 +225,21 @@ module.exports = (repo) => { }) it('resolver.get round-trip', async () => { - const resultPut = resolver.put([blobNode], multicodec.GIT_RAW) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const node = await resultGet.first() + const cid = await resolver.put(blobNode, multicodec.GIT_RAW) + const node = await resolver.get(cid) expect(node).to.deep.equal(blobNode) }) it('resolver.remove', async () => { - const resultPut = resolver.put([blobNode], multicodec.GIT_RAW) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const sameAsBlobNode = await resultGet.first() + const cid = await resolver.put(blobNode, multicodec.GIT_RAW) + const sameAsBlobNode = await resolver.get(cid) expect(sameAsBlobNode).to.deep.equal(blobNode) return remove() async function remove () { - const resultRemove = resolver.remove([cid]) - // The items are deleted through iteration - await resultRemove.last() + await resolver.remove(cid) // Verify that the item got really deleted - const resultGet = resolver.get([cid]) - await expect(resultGet.next()).to.eventually.be.rejected() + await expect(resolver.get(cid)).to.eventually.be.rejected() } }) }) diff --git a/test/ipld-zcash.js b/test/ipld-zcash.js index b346258..9d9de42 100644 --- a/test/ipld-zcash.js +++ b/test/ipld-zcash.js @@ -89,7 +89,7 @@ module.exports = (repo) => { async function store () { const nodes = [node1, node2, node3] - const result = resolver.put(nodes, multicodec.ZCASH_BLOCK) + const result = resolver.putMany(nodes, multicodec.ZCASH_BLOCK) ;[cid1, cid2, cid3] = await result.all() done() @@ -98,8 +98,7 @@ module.exports = (repo) => { describe('public api', () => { it('resolver.put with format', async () => { - const result = resolver.put([node1], multicodec.ZCASH_BLOCK) - const cid = await result.first() + const cid = await resolver.put(node1, multicodec.ZCASH_BLOCK) expect(cid.version).to.equal(1) expect(cid.codec).to.equal('zcash-block') expect(cid.multihash).to.exist() @@ -108,10 +107,9 @@ module.exports = (repo) => { }) it('resolver.put with format + hashAlg', async () => { - const result = resolver.put([node1], multicodec.ZCASH_BLOCK, { + const cid = await resolver.put(node1, multicodec.ZCASH_BLOCK, { hashAlg: multicodec.SHA3_512 }) - const cid = await result.first() expect(cid.version).to.equal(1) expect(cid.codec).to.equal('zcash-block') expect(cid.multihash).to.exist() @@ -152,28 +150,21 @@ module.exports = (repo) => { }) it('resolver.get round-trip', async () => { - const resultPut = resolver.put([node1], multicodec.ZCASH_BLOCK) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const node = await resultGet.first() + const cid = await resolver.put(node1, multicodec.ZCASH_BLOCK) + const node = await resolver.get(cid) expect(node.toString()).to.deep.equal(node1.toString()) }) it('resolver.remove', async () => { - const resultPut = resolver.put([node1], multicodec.ZCASH_BLOCK) - const cid = await resultPut.first() - const resultGet = resolver.get([cid]) - const sameAsNode1 = await resultGet.first() + const cid = await resolver.put(node1, multicodec.ZCASH_BLOCK) + const sameAsNode1 = await resolver.get(cid) expect(sameAsNode1).to.deep.equal(node1) return remove() async function remove () { - const resultRemove = resolver.remove([cid]) - // The items are deleted through iteration - await resultRemove.last() + await resolver.remove(cid) // Verify that the item got really deleted - const resultGet = resolver.get([cid]) - await expect(resultGet.next()).to.eventually.be.rejected() + await expect(resolver.get(cid)).to.eventually.be.rejected() } }) })