Merge pull request #420 from Inist-CNRS/tar-extract-buffer

Tar extract buffer

packages/basics/src/file-merge.js

@@ -0,0 +1,52 @@
+/**
+ * Take `Object` or `Buffer` and throw only one document
+ *
+ * ```json
+ * [ fi1e1.csv, file2.csv ]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [use]
+ * plugin = basics
+ *
+ * [FILELoad]
+ * [FILEMerge]
+ * [replace]
+ * path = contentOfFile1AndFile2
+ * value = self()
+ *
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [
+ * (...)
+ * ]
+ * ```
+ *
+ * @name FILEMerge
+ * @returns {Object}
+ */
+export default function FILEMerge(data, feed) {
+    if (this.isFirst()) {
+        this.chunks = [];
+        this.length = 0;
+        this.isBuffer = Buffer.isBuffer(data);
+    }
+    if (this.isLast()) {
+        feed.write(this.isBuffer ? Buffer.concat(this.chunks, this.length) : this.chunks.join(''));
+        feed.close();
+        return;
+    }
+
+    if (this.isBuffer) {
+        this.length += data.length;
+    } else {
+        this.length = this.chunks.length;
+    }
+    this.chunks.push(data);
+    feed.end();
+}

packages/basics/src/index.js

@@ -31,6 +31,7 @@ import TARDump from './tar-dump';
 import INIString from './ini-string';
 import FILESave from './file-save';
 import FILELoad from './file-load';
+import FILEMerge from './file-merge';
 
 const funcs = {
     BUFObject,
@@ -66,6 +67,7 @@ const funcs = {
     INIString,
     FILESave,
     FILELoad,
+    FILEMerge,
     // aliases
     bufferify: BUFObject.BUFObject,
     concat: TXTConcat.TXTConcat,

packages/basics/src/tar-extract.js

@@ -19,14 +19,16 @@ import writeTo from 'stream-write';
  * @name TARExtract
  * @param {String} [path="**\/*.json"] Regex to select the files to extract
  * @param {String} [json=true] Parse as JSON the content of each file
+ * @param {Boolean} [text=true] The content of each file is converted to a string (otherwise it remains a buffer)
  * @param {Boolean} [compress=false] Enable gzip compression
  * @returns {{id: String, value: String}[]}
  */
 export default function TARExtract(data, feed) {
     const filesPatern = this.getParam('path', '**/*.json');
     if (this.isFirst()) {
         const { ezs } = this;
-        const json = this.getParam('json', true);
+        const text = this.getParam('text', true);
+        const json = text ? this.getParam('json', true) : false;
         const compress = this.getParam('compress', false);
         this.input = ezs.createStream(ezs.objectMode());
         this.output = ezs.createStream(ezs.objectMode());
@@ -35,15 +37,24 @@ export default function TARExtract(data, feed) {
             extract.on('entry', async (header, stream, next) => {
                 if (micromatch.isMatch(header.name, filesPatern)) {
                     try {
-                        const contentRaw = await getStream(stream);
                         if (json) {
-                            const contentJson = JSON.parse(contentRaw);
+                            const contentText = await getStream(stream);
+                            const contentJson = JSON.parse(contentText);
                             return writeTo(
                                 this.output,
                                 contentJson,
                                 () => next(),
                             );
                         }
+                        if (text) {
+                            const contentText = await getStream(stream);
+                            return writeTo(
+                                this.output,
+                                { id: header.name, value: contentText },
+                                () => next(),
+                            );
+                        }
+                        const contentRaw = await getStream(stream, { encoding: 'buffer' });
                         return writeTo(
                             this.output,
                             { id: header.name, value: contentRaw },

packages/basics/test/file-merge.js

@@ -0,0 +1,56 @@
+import fs from 'fs';
+import assert from 'assert';
+import from from 'from';
+import ezs from '../../core/src';
+import statements from '../src';
+import basics from '../../basics/src';
+
+ezs.addPath(__dirname);
+
+describe('FILEMerge', () => {
+    test('with existing file (buffer)', (done) => {
+        ezs.use(statements);
+        ezs.use(basics);
+        const input = [
+            'data01.json',
+        ];
+        const output = [];
+        from(input)
+            .pipe(ezs('FILELoad', { location: __dirname }))
+            .pipe(ezs('FILEMerge', { location: __dirname }))
+            .pipe(ezs.catch())
+            .on('error', done)
+            .on('data', (chunk) => {
+                output.push(JSON.parse(String(chunk)));
+            })
+            .on('end', () => {
+                assert.equal(output.length, 1);
+                assert.equal(output[0][0].id, '9.928');
+                assert.equal(output[0][0].value, 1);
+                done();
+            });
+    });
+    test('with existing file (string)', (done) => {
+        ezs.use(statements);
+        ezs.use(basics);
+        const input = [
+            'data01.json',
+        ];
+        const output = [];
+        from(input)
+            .pipe(ezs('FILELoad', { location: __dirname }))
+            .pipe(ezs('TXTParse'))
+            .pipe(ezs('FILEMerge', { location: __dirname }))
+            .pipe(ezs.catch())
+            .on('error', done)
+            .on('data', (chunk) => {
+                output.push(JSON.parse(chunk));
+            })
+            .on('end', () => {
+                assert.equal(output.length, 1);
+                assert.equal(output[0][0].id, '9.928');
+                assert.equal(output[0][0].value, 1);
+                done();
+            });
+    });
+});

packages/basics/test/tar-extract.js

@@ -67,6 +67,22 @@ describe('TARExtract', () => {
                 done();
             });
     });
+    it('should extract Binary content', (done) => {
+        const result = [];
+        fs.createReadStream('./packages/basics/examples/data/test.tar')
+            .pipe(ezs('TARExtract', { path: '**/*.txt', text: false }))
+            .pipe(ezs.catch())
+            .on('data', (chunk) => {
+                const str = chunk.value.toString();
+                assert.equal(str, 'ok\n');
+                result.push(str);
+            })
+            .on('error', done)
+            .on('end', () => {
+                assert.equal(result.length, 10);
+                done();
+            });
+    });
     it('should ignore Wrong JSON files', (done) => {
         const result = [];
         fs.createReadStream('./packages/basics/examples/data/test2.tar')

packages/core/package.json

@@ -46,7 +46,7 @@
   "directories": {
     "test": "test"
   },
-  "gitHead": "bc1be6636627b450c72d59ec404c43d87d7a42aa",
+  "gitHead": "483fb747db3ae5d72ffeb49dc29fa44fe4dbde84",
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/core#readme",
   "keywords": [
     "stream",