From 086b49f7b3c24f551584514869bbe9ec135f7d0a Mon Sep 17 00:00:00 2001 From: Nicolas Thouvenin Date: Fri, 25 Aug 2023 15:54:55 +0200 Subject: [PATCH 1/6] =?UTF-8?q?fix:=20=F0=9F=90=9B=20tar=20extract=20freez?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/basics/src/tar-extract.js | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/basics/src/tar-extract.js b/packages/basics/src/tar-extract.js index b4273492..f0956c32 100644 --- a/packages/basics/src/tar-extract.js +++ b/packages/basics/src/tar-extract.js @@ -34,21 +34,28 @@ export default function TARExtract(data, feed) { this.whenEnd = new Promise((resolve, reject) => { extract.on('entry', async (header, stream, next) => { if (micromatch.isMatch(header.name, filesPatern)) { - const contentRaw = await getStream(stream); - if (json) { - const contentJson = JSON.parse(contentRaw); + try { + const contentRaw = await getStream(stream); + if (json) { + const contentJson = JSON.parse(contentRaw); + return writeTo( + this.output, + contentJson, + () => next(), + ); + } return writeTo( this.output, - contentJson, + { id: header.name, value: contentRaw }, () => next(), ); + } catch (e) { + console.warn(`WARNING: file was ignored (${header.name})`, e); + stream.resume(); + return next(); } - return writeTo( - this.output, - { id: header.name, value: contentRaw }, - () => next(), - ); } + stream.resume(); return next(); }); extract.on('error', reject); From 8905ff8aa82838ec708f09d8a2294ce1502cdf22 Mon Sep 17 00:00:00 2001 From: Nicolas Thouvenin Date: Fri, 25 Aug 2023 16:17:27 +0200 Subject: [PATCH 2/6] =?UTF-8?q?test:=20=F0=9F=92=8D=20increase=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/basics/examples/data/test2.tar | Bin 0 -> 10240 bytes packages/basics/test/tar-extract.js | 14 ++++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 packages/basics/examples/data/test2.tar diff --git a/packages/basics/examples/data/test2.tar b/packages/basics/examples/data/test2.tar new file mode 100644 index 0000000000000000000000000000000000000000..b5cb4e81c51a00cdc89da7af84935b3de7504eea GIT binary patch literal 10240 zcmeH}?Fxc06o&WPs}L`scJ>L~ji8W48M%UpuBZFx%GxB#KavTvf_Hxg&Uwc3o@eOF z#)&zLI=CO!YUU@hn!0mX3WJeSR*;n3D8@-xWm&4Dp6c=OL*w#&nliVn565y_ZF`Eo zTkqNS>yKTVf04U9UK50XhvqMh@cBz)2>DOZJDvQ?&CZ=>tETQ#VC;E9{w8=mz<(m{ z^Vi0B|M}R0pX`JG{{CG;hU5C8!X009sH0T2KI5C8!X a009sH0T2KI5C8!X009sH0T2Lzm;~;8Aci&o literal 0 HcmV?d00001 diff --git a/packages/basics/test/tar-extract.js b/packages/basics/test/tar-extract.js index f8a8b915..6f110fd5 100644 --- a/packages/basics/test/tar-extract.js +++ b/packages/basics/test/tar-extract.js @@ -67,4 +67,18 @@ describe('TARExtract', () => { done(); }); }); + it('should ignore Wrong JSON files', (done) => { + const result = []; + fs.createReadStream('./packages/basics/examples/data/test2.tar') + .pipe(ezs('TARExtract', { path: '**/*.json', json: true, compress: false})) + .pipe(ezs.catch()) + .on('data', (chunk) => { + result.push(chunk); + }) + .on('error', done) + .on('end', () => { + assert.equal(result.length, 1); + done(); + }); + }); }); From 3699fdbdf5ba6853d150ea82bbfe3e72bea1979d Mon Sep 17 00:00:00 2001 From: Nicolas Thouvenin Date: Fri, 25 Aug 2023 16:24:09 +0200 Subject: [PATCH 3/6] Publish - @ezs/basics@2.5.3 --- packages/basics/CHANGELOG.md | 11 +++++++++++ packages/basics/package-lock.json | 2 +- packages/basics/package.json | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/basics/CHANGELOG.md b/packages/basics/CHANGELOG.md index 2dd8f7cb..5da7c848 100644 --- a/packages/basics/CHANGELOG.md +++ b/packages/basics/CHANGELOG.md @@ -3,6 +3,17 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [2.5.3](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.5.2...@ezs/basics@2.5.3) (2023-08-25) + + +### Bug Fixes + +* 🐛 tar extract freeze ([086b49f](https://github.com/Inist-CNRS/ezs/commit/086b49f7b3c24f551584514869bbe9ec135f7d0a)) + + + + + ## [2.5.2](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.5.1...@ezs/basics@2.5.2) (2023-08-25) **Note:** Version bump only for package @ezs/basics diff --git a/packages/basics/package-lock.json b/packages/basics/package-lock.json index eb366a5a..c0dda3a8 100644 --- a/packages/basics/package-lock.json +++ b/packages/basics/package-lock.json @@ -1,6 +1,6 @@ { "name": "@ezs/basics", - "version": "2.5.2", + "version": "2.5.3", "lockfileVersion": 2, "requires": true, "packages": { diff --git a/packages/basics/package.json b/packages/basics/package.json index 1805c431..4b238a22 100644 --- a/packages/basics/package.json +++ b/packages/basics/package.json @@ -1,7 +1,7 @@ { "name": "@ezs/basics", "description": "Basics statements for EZS", - "version": "2.5.2", + "version": "2.5.3", "author": "Nicolas Thouvenin ", "bugs": "https://github.com/Inist-CNRS/ezs/issues", "dependencies": { From 961e20157816b3792ff6ee692e8fd5a6b437968c Mon Sep 17 00:00:00 2001 From: Nicolas Thouvenin Date: Fri, 8 Sep 2023 13:53:18 +0200 Subject: [PATCH 4/6] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20add=20identifier=20p?= =?UTF-8?q?arameter=20to=20[graph]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/analytics/src/graph.js | 13 ++++++------- packages/analytics/test/network.js | 26 +++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/packages/analytics/src/graph.js b/packages/analytics/src/graph.js index 499fe0f6..0af89846 100644 --- a/packages/analytics/src/graph.js +++ b/packages/analytics/src/graph.js @@ -45,22 +45,21 @@ export default function graph(data, feed) { feed.close(); return; } - let fields = this.getParam('path', []); - if (!Array.isArray(fields)) { - fields = [fields]; - } - - const values = fields + const path = this.getParam('path', []); + const idt = this.getParam('identifier', false); + const weight = idt === false ? 1 : get(data, idt, 1); + const values = [].concat(path) .map((key) => get(data, key)) .filter((x) => x) .map((item) => (item instanceof Array ? item : [item])) .reduce((pre, cur) => pre.concat(cur), []) + .filter(Boolean) .sort(); values.forEach( (v, i) => values .slice(i + 1) - .forEach((w) => feed.write(core([v, w], 1))), + .forEach((w) => feed.write(core([v, w], weight))), ); feed.end(); } diff --git a/packages/analytics/test/network.js b/packages/analytics/test/network.js index d8163370..3537e539 100644 --- a/packages/analytics/test/network.js +++ b/packages/analytics/test/network.js @@ -6,7 +6,7 @@ import statements from '../src'; ezs.addPath(__dirname); describe('network', () => { - it('graph', (done) => { + it('graph #1', (done) => { ezs.use(statements); const res = []; from([ @@ -30,6 +30,30 @@ describe('network', () => { }); }); + it('graph #2', (done) => { + ezs.use(statements); + const res = []; + from([ + { i: 'doc#1', a: ['x', 'b', 'z'] }, + { i: 'doc#2', a: ['t', 'b', 'z'] }, + { i: 'doc#3', a: ['t', 'c', 'z'] }, + { i: 'doc#4', a: ['y', 'd', 'z'] }, + { i: 'doc#5', a: ['x', 'b', 'z'] }, + ]) + .pipe(ezs('graph', { path: 'a', identifier: 'i' })) + .pipe(ezs('reducing')) + .on('data', (chunk) => { + assert(typeof chunk === 'object'); + res.push(chunk); + }) + .on('end', () => { + assert.equal(10, res.length); + assert.equal(2, res[1].value.length); + done(); + }); + }); + + it('segment #3', (done) => { ezs.use(statements); const res = []; From 49c215072dc275614d168800e54e326b99e37758 Mon Sep 17 00:00:00 2001 From: Nicolas Thouvenin Date: Fri, 8 Sep 2023 14:21:56 +0200 Subject: [PATCH 5/6] =?UTF-8?q?docs:=20=E2=9C=8F=EF=B8=8F=20update=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin-analytics.md | 3 ++- packages/analytics/README.md | 3 ++- packages/analytics/src/graph.js | 1 + packages/analytics/src/segment.js | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/plugin-analytics.md b/docs/plugin-analytics.md index 54d6ef13..1cf8d07c 100644 --- a/docs/plugin-analytics.md +++ b/docs/plugin-analytics.md @@ -497,6 +497,7 @@ Output: #### Parameters - `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** +- `identifier` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** path to use to set value result field (if not set or not exists, 1 is use as a default value) (optional, default `false`) Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** @@ -1248,7 +1249,7 @@ Output: - `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** path (optional, default `value`) - `aggregate` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** aggregate all values for all paths (or not) (optional, default `true`) -- `identifier` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** path to use to set value result field (if not set or not exists, 1 is use as a default value) (optional, default `false`) +- `identifier` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** path to use to set value result field (if not set or not exists, 1 is use as a default value) (optional, default `false`) Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** diff --git a/packages/analytics/README.md b/packages/analytics/README.md index 54d6ef13..1cf8d07c 100644 --- a/packages/analytics/README.md +++ b/packages/analytics/README.md @@ -497,6 +497,7 @@ Output: #### Parameters - `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** +- `identifier` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** path to use to set value result field (if not set or not exists, 1 is use as a default value) (optional, default `false`) Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** @@ -1248,7 +1249,7 @@ Output: - `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** path (optional, default `value`) - `aggregate` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** aggregate all values for all paths (or not) (optional, default `true`) -- `identifier` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** path to use to set value result field (if not set or not exists, 1 is use as a default value) (optional, default `false`) +- `identifier` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** path to use to set value result field (if not set or not exists, 1 is use as a default value) (optional, default `false`) Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** diff --git a/packages/analytics/src/graph.js b/packages/analytics/src/graph.js index 0af89846..6f1b44b5 100644 --- a/packages/analytics/src/graph.js +++ b/packages/analytics/src/graph.js @@ -38,6 +38,7 @@ import core from './core'; * * @name graph * @param {String} path + * @param {String} [identifier=false] path to use to set value result field (if not set or not exists, 1 is use as a default value) * @returns {Object} */ export default function graph(data, feed) { diff --git a/packages/analytics/src/segment.js b/packages/analytics/src/segment.js index 41f5615a..851ba9fd 100644 --- a/packages/analytics/src/segment.js +++ b/packages/analytics/src/segment.js @@ -80,7 +80,7 @@ import core from './core'; * @name segment * @param {String} [path=value] path * @param {Boolean} [aggregate=true] aggregate all values for all paths (or not) - * @param {Boolean} [identifier=false] path to use to set value result field (if not set or not exists, 1 is use as a default value) + * @param {String} [identifier=false] path to use to set value result field (if not set or not exists, 1 is use as a default value) * @returns {Object} */ export default function segment(data, feed) { From c34d2b5e4ed6071da40a5986cab4a7254093809a Mon Sep 17 00:00:00 2001 From: Nicolas Thouvenin Date: Fri, 8 Sep 2023 14:22:47 +0200 Subject: [PATCH 6/6] Publish - @ezs/analytics@2.2.0 --- packages/analytics/CHANGELOG.md | 11 +++++++++++ packages/analytics/package-lock.json | 2 +- packages/analytics/package.json | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/analytics/CHANGELOG.md b/packages/analytics/CHANGELOG.md index 6e9a01e5..f5594fed 100644 --- a/packages/analytics/CHANGELOG.md +++ b/packages/analytics/CHANGELOG.md @@ -3,6 +3,17 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +# [2.2.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/analytics@2.1.0...@ezs/analytics@2.2.0) (2023-09-08) + + +### Features + +* 🎸 add identifier parameter to [graph] ([961e201](https://github.com/Inist-CNRS/ezs/commit/961e20157816b3792ff6ee692e8fd5a6b437968c)) + + + + + # [2.1.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/analytics@2.0.23...@ezs/analytics@2.1.0) (2023-08-25) diff --git a/packages/analytics/package-lock.json b/packages/analytics/package-lock.json index a78ebdec..0a72f9f7 100644 --- a/packages/analytics/package-lock.json +++ b/packages/analytics/package-lock.json @@ -1,6 +1,6 @@ { "name": "@ezs/analytics", - "version": "2.1.0", + "version": "2.2.0", "lockfileVersion": 2, "requires": true, "packages": { diff --git a/packages/analytics/package.json b/packages/analytics/package.json index a9bf95b5..864f4a2b 100644 --- a/packages/analytics/package.json +++ b/packages/analytics/package.json @@ -1,7 +1,7 @@ { "name": "@ezs/analytics", "description": "Analytics statements for EZS", - "version": "2.1.0", + "version": "2.2.0", "author": "Nicolas Thouvenin ", "bugs": "https://github.com/Inist-CNRS/ezs/issues", "dependencies": {