From 7ab7e331329303eead61041ef956f1a4d2fa12cf Mon Sep 17 00:00:00 2001 From: reiji-h Date: Fri, 1 Nov 2024 08:55:49 +0000 Subject: [PATCH 01/22] on error close --- .../server/services/growi-plugin/growi-plugin.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts index ec1743ae838..6bdbc7cafa4 100644 --- a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts +++ b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts @@ -209,8 +209,10 @@ export class GrowiPluginService implements IGrowiPluginService { private async unzip(zipFilePath: fs.PathLike, destPath: fs.PathLike): Promise { try { - const stream = fs.createReadStream(zipFilePath); - const unzipFileStream = stream.pipe(unzipStream.Extract({ path: destPath.toString() })); + const readZipStream = fs.createReadStream(zipFilePath); + const writeUnZipStream = unzipStream.Extract({ path: destPath.toString() }); + + const unzipFileStream = readZipStream.on('error', () => { writeUnZipStream.end() }).pipe(writeUnZipStream).on('error', () => { readZipStream.destroy() }); await streamToPromise(unzipFileStream); } From 1a06229b70f4283f8a5ed7efd8f7a99313d7aea3 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Wed, 6 Nov 2024 05:05:37 +0000 Subject: [PATCH 02/22] add on error for stream pipe --- .../app/src/server/routes/apiv3/page/index.ts | 5 +- apps/app/src/server/service/export.js | 22 ++- .../src/server/service/growi-bridge/index.ts | 4 +- apps/app/src/server/service/import/import.ts | 32 ++- .../delete-completely-user-home-by-system.ts | 19 +- apps/app/src/server/service/page/index.ts | 186 +++++++++++++++--- 6 files changed, 236 insertions(+), 32 deletions(-) diff --git a/apps/app/src/server/routes/apiv3/page/index.ts b/apps/app/src/server/routes/apiv3/page/index.ts index 3728d707883..62ef10d9916 100644 --- a/apps/app/src/server/routes/apiv3/page/index.ts +++ b/apps/app/src/server/routes/apiv3/page/index.ts @@ -1,4 +1,5 @@ import path from 'path'; +import type { Readable } from 'stream'; import type { IPage } from '@growi/core'; import { @@ -734,7 +735,7 @@ module.exports = (crowi) => { fileName = '_top'; } - let stream; + let stream: Readable; try { stream = exportService.getReadStreamFromRevision(revision, format); @@ -759,7 +760,7 @@ module.exports = (crowi) => { }; await crowi.activityService.createActivity(parameters); - return stream.pipe(res); + return stream.pipe(res).on('error', () => { stream.destroy() }); }); /** diff --git a/apps/app/src/server/service/export.js b/apps/app/src/server/service/export.js index 78b947cfbde..cb928409301 100644 --- a/apps/app/src/server/service/export.js +++ b/apps/app/src/server/service/export.js @@ -197,9 +197,29 @@ class ExportService { const writeStream = fs.createWriteStream(jsonFileToWrite, { encoding: this.growiBridgeService.getEncoding() }); readStream + .on('error', () => { + logStream.end(); + transformStream.end(); + writeStream.end(); + }) .pipe(logStream) + .on('error', () => { + readStream.destroy(); + transformStream.end(); + writeStream.end(); + }) .pipe(transformStream) - .pipe(writeStream); + .on('error', () => { + readStream.destroy(); + logStream.destroy(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destroy(); + logStream.destroy(); + readStream.destroy(); + }); await streamToPromise(writeStream); diff --git a/apps/app/src/server/service/growi-bridge/index.ts b/apps/app/src/server/service/growi-bridge/index.ts index e0c673c3817..59aab17b22a 100644 --- a/apps/app/src/server/service/growi-bridge/index.ts +++ b/apps/app/src/server/service/growi-bridge/index.ts @@ -78,7 +78,9 @@ class GrowiBridgeService { let meta = {}; const readStream = fs.createReadStream(zipFile); - const unzipStreamPipe = readStream.pipe(unzipStream.Parse()); + const parseStream = unzipStream.Parse(); + const unzipStreamPipe = readStream.on('error', () => parseStream.end()).pipe(parseStream).on('error', () => readStream.destroy()); + let tapPromise; const unzipEntryStream = unzipStreamPipe.on('entry', (entry: Entry) => { diff --git a/apps/app/src/server/service/import/import.ts b/apps/app/src/server/service/import/import.ts index 00875ce013f..3af2acaa0af 100644 --- a/apps/app/src/server/service/import/import.ts +++ b/apps/app/src/server/service/import/import.ts @@ -268,10 +268,40 @@ export class ImportService { }); readStream + .on('error', () => { + jsonStream.end(); + convertStream.end(); + batchStream.end(); + writeStream.end(); + }) .pipe(jsonStream) + .on('error', () => { + readStream.destroy(); + convertStream.end(); + batchStream.end(); + writeStream.end(); + }) .pipe(convertStream) + .on('error', () => { + readStream.destroy(); + jsonStream.destroy(); + batchStream.end(); + writeStream.end(); + }) .pipe(batchStream) - .pipe(writeStream); + .on('error', () => { + readStream.destroy(); + jsonStream.destroy(); + convertStream.destroy(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destroy(); + jsonStream.destroy(); + convertStream.destroy(); + batchStream.destroy(); + }); await streamToPromise(writeStream); diff --git a/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts b/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts index c39f547fd0b..fcebae418c7 100644 --- a/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts +++ b/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts @@ -87,8 +87,9 @@ export const deleteCompletelyUserHomeBySystem = async(userHomepagePath: string, .lean() .cursor({ batchSize: BULK_REINDEX_SIZE }); - let count = 0; + const batchStream = createBatchStream(BULK_REINDEX_SIZE); + let count = 0; const writeStream = new Writable({ objectMode: true, async write(batch, encoding, callback) { @@ -110,8 +111,20 @@ export const deleteCompletelyUserHomeBySystem = async(userHomepagePath: string, }); readStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(writeStream); + .on('error', () => { + batchStream.end(); + writeStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + readStream.destroy(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destroy(); + batchStream.destroy(); + }); await streamToPromise(writeStream); // ────────┤ end │───────── diff --git a/apps/app/src/server/service/page/index.ts b/apps/app/src/server/service/page/index.ts index ff49ca9de00..3938984e100 100644 --- a/apps/app/src/server/service/page/index.ts +++ b/apps/app/src/server/service/page/index.ts @@ -1006,6 +1006,8 @@ class PageService implements IPageService { const factory = new PageCursorsForDescendantsFactory(user, targetPage, true); const readStream = await factory.generateReadable(); + const batchStream = createBatchStream(BULK_REINDEX_SIZE); + const newPagePathPrefix = newPagePath; const pathRegExp = new RegExp(`^${escapeStringRegexp(targetPage.path)}`, 'i'); @@ -1044,15 +1046,27 @@ class PageService implements IPageService { }); readStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(writeStream); - + .on('error', () => { + batchStream.end(); + writeStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + readStream.destroy(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destroy(); + batchStream.destroy(); + }); await streamToPromise(writeStream); } private async renameDescendantsWithStreamV4(targetPage, newPagePath, user, options = {}) { const readStream = await this.generateReadStreamToOperateOnlyDescendants(targetPage.path, user); + const batchStream = createBatchStream(BULK_REINDEX_SIZE); const newPagePathPrefix = newPagePath; const pathRegExp = new RegExp(`^${escapeStringRegexp(targetPage.path)}`, 'i'); @@ -1084,9 +1098,20 @@ class PageService implements IPageService { }); readStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(writeStream); - + .on('error', () => { + batchStream.end(); + writeStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + readStream.destroy(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destroy(); + batchStream.destroy(); + }); await streamToPromise(writeStream); } @@ -1469,6 +1494,7 @@ class PageService implements IPageService { const iterableFactory = new PageCursorsForDescendantsFactory(user, page, true); const readStream = await iterableFactory.generateReadable(); + const batchStream = createBatchStream(BULK_REINDEX_SIZE); const newPagePathPrefix = newPagePath; const pathRegExp = new RegExp(`^${escapeStringRegexp(page.path)}`, 'i'); @@ -1502,8 +1528,20 @@ class PageService implements IPageService { }); readStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(writeStream); + .on('error', () => { + batchStream.end(); + writeStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + readStream.destroy(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destroy(); + batchStream.destroy(); + }); await streamToPromise(writeStream); @@ -1512,6 +1550,7 @@ class PageService implements IPageService { private async duplicateDescendantsWithStreamV4(page, newPagePath, user, onlyDuplicateUserRelatedResources: boolean) { const readStream = await this.generateReadStreamToOperateOnlyDescendants(page.path, user); + const batchStream = createBatchStream(BULK_REINDEX_SIZE); const newPagePathPrefix = newPagePath; const pathRegExp = new RegExp(`^${escapeStringRegexp(page.path)}`, 'i'); @@ -1543,8 +1582,20 @@ class PageService implements IPageService { }); readStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(writeStream); + .on('error', () => { + batchStream.end(); + writeStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + readStream.destory(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destory(); + batchStream.destroy(); + }); await streamToPromise(writeStream); @@ -1841,6 +1892,7 @@ class PageService implements IPageService { readStream = await factory.generateReadable(); } + const batchStream = createBatchStream(BULK_REINDEX_SIZE); const deleteDescendants = this.deleteDescendants.bind(this); let count = 0; @@ -1874,8 +1926,20 @@ class PageService implements IPageService { }); readStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(writeStream); + .on('error', () => { + batchStream.end(); + writeStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + readStream.destory(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destory(); + batchStream.destroy(); + }); await streamToPromise(writeStream); @@ -2105,6 +2169,8 @@ class PageService implements IPageService { readStream = await factory.generateReadable(); } + const batchStream = createBatchStream(BULK_REINDEX_SIZE); + let count = 0; let nDeletedNonEmptyPages = 0; // used for updating descendantCount @@ -2137,8 +2203,20 @@ class PageService implements IPageService { }); readStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(writeStream); + .on('error', () => { + batchStream.end(); + writeStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + readStream.destory(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destory(); + batchStream.destroy(); + }); await streamToPromise(writeStream); @@ -2416,7 +2494,7 @@ class PageService implements IPageService { ); const childPagesReadableStream = builder.query.cursor({ batchSize: BULK_REINDEX_SIZE }); - + const batchStream = createBatchStream(BULK_REINDEX_SIZE); const childPagesWritable = new Writable({ objectMode: true, write: async(batch, encoding, callback) => { @@ -2426,8 +2504,20 @@ class PageService implements IPageService { }); childPagesReadableStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(childPagesWritable); + .on('error', () => { + batchStream.end(); + childPagesWritable.end(); + }) + .pipe(batchStream) + .on('error', () => { + childPagesReadableStream.destroy(); + childPagesWritable.end(); + }) + .pipe(childPagesWritable) + .on('error', () => { + childPagesReadableStream.destroy(); + childPagesWritable.destroy(); + }); await streamToPromise(childPagesWritable); } @@ -2465,6 +2555,7 @@ class PageService implements IPageService { } const readStream = await this.generateReadStreamToOperateOnlyDescendants(targetPage.path, user); + const batchStream = createBatchStream(BULK_REINDEX_SIZE); const revertDeletedDescendants = this.revertDeletedDescendants.bind(this); let count = 0; @@ -2494,8 +2585,20 @@ class PageService implements IPageService { }); readStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(writeStream); + .on('error', () => { + batchStream.end(); + writeStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + readStream.destroy(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => { + readStream.destroy(); + batchStream.destroy(); + }); await streamToPromise(writeStream); @@ -2504,6 +2607,7 @@ class PageService implements IPageService { private async revertDeletedDescendantsWithStreamV4(targetPage, user, options = {}) { const readStream = await this.generateReadStreamToOperateOnlyDescendants(targetPage.path, user); + const batchStream = createBatchStream(BULK_REINDEX_SIZE); const revertDeletedDescendants = this.revertDeletedDescendants.bind(this); let count = 0; @@ -2529,8 +2633,17 @@ class PageService implements IPageService { }); readStream - .pipe(createBatchStream(BULK_REINDEX_SIZE)) - .pipe(writeStream); + .on('error', () => { + batchStream.end(); + writeStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + readStream.destory(); + writeStream.end(); + }) + .pipe(writeStream) + .on('error', () => {}); await streamToPromise(readStream); @@ -3379,8 +3492,20 @@ class PageService implements IPageService { }); pagesStream + .on('error', () => { + batchStream.end(); + migratePagesStream.end(); + }) .pipe(batchStream) - .pipe(migratePagesStream); + .on('error', () => { + pagesStream.destroy(); + migratePagesStream.end(); + }) + .pipe(migratePagesStream) + .on('error', () => { + pagesStream.destroy(); + batchStream.destroy(); + }); await streamToPromise(migratePagesStream); @@ -3483,6 +3608,7 @@ class PageService implements IPageService { */ async recountAndUpdateDescendantCountOfPages(pageCursor: Cursor, batchSize:number): Promise { const Page = this.crowi.model('Page'); + const batchStream = createBatchStream(batchSize); const recountWriteStream = new Writable({ objectMode: true, async write(pageDocuments, encoding, callback) { @@ -3497,8 +3623,20 @@ class PageService implements IPageService { }, }); pageCursor - .pipe(createBatchStream(batchSize)) - .pipe(recountWriteStream); + .on('error', () => { + batchStream.end(); + recountWriteStream.end(); + }) + .pipe(batchStream) + .on('error', () => { + pageCursor.destroy(); + recountWriteStream.end(); + }) + .pipe(recountWriteStream) + .on('error', () => { + pageCursor.destroy(); + batchStream.destroy(); + }); await streamToPromise(recountWriteStream); } From efeb0f18d8b6a1827123cf103f84e43a750749ac Mon Sep 17 00:00:00 2001 From: reiji-h Date: Wed, 6 Nov 2024 06:06:54 +0000 Subject: [PATCH 03/22] explicitly close Stream --- apps/app/src/server/service/export.js | 5 ++++- .../service/search-delegator/elasticsearch.ts | 22 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/apps/app/src/server/service/export.js b/apps/app/src/server/service/export.js index cb928409301..77f0e7c96b4 100644 --- a/apps/app/src/server/service/export.js +++ b/apps/app/src/server/service/export.js @@ -375,7 +375,10 @@ class ExportService { const output = fs.createWriteStream(zipFile); // pipe archive data to the file - archive.pipe(output); + archive + .on('error', () => { output.end() }) + .pipe(output) + .on('error', () => { archive.destroy() }); // finalize the archive (ie we are done appending files but streams have to finish yet) // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand diff --git a/apps/app/src/server/service/search-delegator/elasticsearch.ts b/apps/app/src/server/service/search-delegator/elasticsearch.ts index 17a89d770f8..9eb5d39b7eb 100644 --- a/apps/app/src/server/service/search-delegator/elasticsearch.ts +++ b/apps/app/src/server/service/search-delegator/elasticsearch.ts @@ -554,11 +554,31 @@ class ElasticsearchDelegator implements SearchDelegator { + batchStream.end(); + appendTagNamesStream.end(); + writeStream.end(); + }) .pipe(batchStream) + .on('error', () => { + readStream.destroy(); + appendTagNamesStream.end(); + writeStream.end(); + }) .pipe(appendTagNamesStream) + .on('error', () => { + readStream.destroy(); + batchStream.destroy(); + writeStream.end(); + }) // .pipe(appendEmbeddingStream) // .pipe(appendFileUploadedStream) - .pipe(writeStream); + .pipe(writeStream) + .on('error', () => { + readStream.destroy(); + batchStream.destroy(); + appendTagNamesStream.destroy(); + }); return streamToPromise(writeStream); } From 878da4ccd6267f4503cc176c69f7f6c7be691590 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Wed, 6 Nov 2024 06:22:32 +0000 Subject: [PATCH 04/22] explicitly close Stream --- .../features/openai/server/services/openai.ts | 14 ++++++++- ...to-page-id-schema-migration--fixed-7549.js | 29 +++++++++++++++++-- .../src/server/service/file-uploader/local.ts | 13 +++++++-- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/apps/app/src/features/openai/server/services/openai.ts b/apps/app/src/features/openai/server/services/openai.ts index 05451668a2e..1d1990d02d2 100644 --- a/apps/app/src/features/openai/server/services/openai.ts +++ b/apps/app/src/features/openai/server/services/openai.ts @@ -343,8 +343,20 @@ class OpenaiService implements IOpenaiService { }); pagesStream + .on('error', () => { + batchStrem.end(); + createVectorStoreFileStream.end(); + }) .pipe(batchStrem) - .pipe(createVectorStoreFileStream); + .on('error', () => { + pagesStream.destroy(); + createVectorStoreFileStream.end(); + }) + .pipe(createVectorStoreFileStream) + .on('error', () => { + pagesStream.destroy(); + batchStrem.destroy(); + }); } async rebuildVectorStore(page: HydratedDocument) { diff --git a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js index c59f3912bea..3f07933ea70 100644 --- a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js +++ b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js @@ -1,6 +1,7 @@ import { Writable } from 'stream'; import mongoose from 'mongoose'; +import { pages } from 'next/dist/build/templates/app-page'; import streamToPromise from 'stream-to-promise'; import getPageModel from '~/server/models/page'; @@ -57,8 +58,20 @@ module.exports = { }); pagesStream + .on('error', () => { + batchStrem.end(); + migratePagesStream.end(); + }) .pipe(batchStrem) - .pipe(migratePagesStream); + .on('error', () => { + pagesStream.destroy(); + migratePagesStream.end(); + }) + .pipe(migratePagesStream) + .on('error', () => { + pagesStream.destroy(); + batchStrem.destroy(); + }); await streamToPromise(migratePagesStream); @@ -108,8 +121,20 @@ module.exports = { }); pagesStream + .on('error', () => { + batchStrem.end(); + migratePagesStream.end(); + }) .pipe(batchStrem) - .pipe(migratePagesStream); + .on('error', () => { + pagesStream.destroy(); + migratePagesStream.end(); + }) + .pipe(migratePagesStream) + .on('error', () => { + pagesStream.destroy(); + batchStrem.destroy(); + }); await streamToPromise(migratePagesStream); diff --git a/apps/app/src/server/service/file-uploader/local.ts b/apps/app/src/server/service/file-uploader/local.ts index ad2b574b1d5..c23b5ae4eea 100644 --- a/apps/app/src/server/service/file-uploader/local.ts +++ b/apps/app/src/server/service/file-uploader/local.ts @@ -163,7 +163,12 @@ module.exports = function(crowi) { // mkdir -p mkdir.sync(dirpath); - const stream = fileStream.pipe(fs.createWriteStream(filePath)); + const writeStream = fs.createWriteStream(filePath); + + const stream = fileStream + .on('error', () => { writeStream.end() }) + .pipe(writeStream) + .on('error', () => { fileStream.destroy() }); return streamToPromise(stream); }; @@ -177,7 +182,11 @@ module.exports = function(crowi) { const fileStream = new Readable(); fileStream.push(data); fileStream.push(null); // EOF - const stream = fileStream.pipe(fs.createWriteStream(absFilePath)); + const writeStream = fs.createWriteStream(absFilePath); + const stream = fileStream + .on('error', () => { writeStream.end() }) + .pipe(writeStream) + .on('error', () => { fileStream.destroy() }); return streamToPromise(stream); }; From ab55c891781643768ed208415a8db57109cb06f1 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Wed, 6 Nov 2024 06:33:34 +0000 Subject: [PATCH 05/22] add error handler --- apps/app/src/server/service/growi-bridge/index.ts | 5 ++++- apps/app/src/server/service/import/import.ts | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/app/src/server/service/growi-bridge/index.ts b/apps/app/src/server/service/growi-bridge/index.ts index 59aab17b22a..80fce77cb17 100644 --- a/apps/app/src/server/service/growi-bridge/index.ts +++ b/apps/app/src/server/service/growi-bridge/index.ts @@ -79,7 +79,10 @@ class GrowiBridgeService { const readStream = fs.createReadStream(zipFile); const parseStream = unzipStream.Parse(); - const unzipStreamPipe = readStream.on('error', () => parseStream.end()).pipe(parseStream).on('error', () => readStream.destroy()); + const unzipStreamPipe = readStream + .on('error', () => parseStream.end()) + .pipe(parseStream) + .on('error', () => readStream.destroy()); let tapPromise; diff --git a/apps/app/src/server/service/import/import.ts b/apps/app/src/server/service/import/import.ts index 3af2acaa0af..afdccf3df3b 100644 --- a/apps/app/src/server/service/import/import.ts +++ b/apps/app/src/server/service/import/import.ts @@ -379,7 +379,11 @@ export class ImportService { */ async unzip(zipFile) { const readStream = fs.createReadStream(zipFile); - const unzipStreamPipe = readStream.pipe(unzipStream.Parse()); + const parseStream = unzipStream.Parse(); + const unzipStreamPipe = readStream + .on('error', () => { parseStream.end() }) + .pipe(parseStream) + .on('error', () => { readStream.destroy() }); const files: string[] = []; unzipStreamPipe.on('entry', (/** @type {Entry} */ entry) => { @@ -400,7 +404,10 @@ export class ImportService { else { const jsonFile = path.join(this.baseDir, fileName); const writeStream = fs.createWriteStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() }); - entry.pipe(writeStream); + entry + .on('error', () => { writeStream.end() }) + .pipe(writeStream) + .on('error', () => { entry.destory() }); files.push(jsonFile); } }); From d663e64147ec6ffd08c4b401bfd6bc5fb3a7d60e Mon Sep 17 00:00:00 2001 From: reiji-h Date: Thu, 7 Nov 2024 05:50:18 +0000 Subject: [PATCH 06/22] fix error --- ...to-page-id-schema-migration--fixed-7549.js | 28 ++----------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js index 3f07933ea70..f7b5657dc06 100644 --- a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js +++ b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js @@ -58,20 +58,8 @@ module.exports = { }); pagesStream - .on('error', () => { - batchStrem.end(); - migratePagesStream.end(); - }) .pipe(batchStrem) - .on('error', () => { - pagesStream.destroy(); - migratePagesStream.end(); - }) - .pipe(migratePagesStream) - .on('error', () => { - pagesStream.destroy(); - batchStrem.destroy(); - }); + .pipe(migratePagesStream); await streamToPromise(migratePagesStream); @@ -121,20 +109,8 @@ module.exports = { }); pagesStream - .on('error', () => { - batchStrem.end(); - migratePagesStream.end(); - }) .pipe(batchStrem) - .on('error', () => { - pagesStream.destroy(); - migratePagesStream.end(); - }) - .pipe(migratePagesStream) - .on('error', () => { - pagesStream.destroy(); - batchStrem.destroy(); - }); + .pipe(migratePagesStream); await streamToPromise(migratePagesStream); From 46d0148ee27f694e69c25c446d3dad788b4567d1 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Thu, 7 Nov 2024 06:47:34 +0000 Subject: [PATCH 07/22] add close stream --- ...to-page-id-schema-migration--fixed-7549.js | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js index f7b5657dc06..3fab4e356e4 100644 --- a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js +++ b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js @@ -1,7 +1,6 @@ import { Writable } from 'stream'; import mongoose from 'mongoose'; -import { pages } from 'next/dist/build/templates/app-page'; import streamToPromise from 'stream-to-promise'; import getPageModel from '~/server/models/page'; @@ -58,8 +57,20 @@ module.exports = { }); pagesStream + .on('error', () => { + batchStrem.end(); + migratePagesStream.end(); + }) .pipe(batchStrem) - .pipe(migratePagesStream); + .on('error', () => { + pagesStream.destory(); + migratePagesStream.end(); + }) + .pipe(migratePagesStream) + .on('error', () => { + pagesStream.destory(); + batchStrem.destory(); + }); await streamToPromise(migratePagesStream); @@ -109,8 +120,20 @@ module.exports = { }); pagesStream + .on('error', () => { + batchStrem.end(); + migratePagesStream.end(); + }) .pipe(batchStrem) - .pipe(migratePagesStream); + .on('error', () => { + pagesStream.destory(); + migratePagesStream.end(); + }) + .pipe(migratePagesStream) + .on('error', () => { + pagesStream.destory(); + batchStrem.destory(); + }); await streamToPromise(migratePagesStream); From 4c7db971b8909a0d61c5fad84bfea76a44714797 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Thu, 7 Nov 2024 07:01:15 +0000 Subject: [PATCH 08/22] clean code --- ...path-to-page-id-schema-migration--fixed-7549.js | 12 ++++++------ apps/app/src/server/service/import/import.ts | 2 +- apps/app/src/server/service/page/index.ts | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js index 3fab4e356e4..d1c158f6dbf 100644 --- a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js +++ b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js @@ -63,13 +63,13 @@ module.exports = { }) .pipe(batchStrem) .on('error', () => { - pagesStream.destory(); + pagesStream.destroy(); migratePagesStream.end(); }) .pipe(migratePagesStream) .on('error', () => { - pagesStream.destory(); - batchStrem.destory(); + pagesStream.destroy(); + batchStrem.destroy(); }); await streamToPromise(migratePagesStream); @@ -126,13 +126,13 @@ module.exports = { }) .pipe(batchStrem) .on('error', () => { - pagesStream.destory(); + pagesStream.destroy(); migratePagesStream.end(); }) .pipe(migratePagesStream) .on('error', () => { - pagesStream.destory(); - batchStrem.destory(); + pagesStream.destroy(); + batchStrem.destroy(); }); await streamToPromise(migratePagesStream); diff --git a/apps/app/src/server/service/import/import.ts b/apps/app/src/server/service/import/import.ts index afdccf3df3b..5f548069fb1 100644 --- a/apps/app/src/server/service/import/import.ts +++ b/apps/app/src/server/service/import/import.ts @@ -407,7 +407,7 @@ export class ImportService { entry .on('error', () => { writeStream.end() }) .pipe(writeStream) - .on('error', () => { entry.destory() }); + .on('error', () => { entry.destroy() }); files.push(jsonFile); } }); diff --git a/apps/app/src/server/service/page/index.ts b/apps/app/src/server/service/page/index.ts index 3938984e100..aeb59defa9f 100644 --- a/apps/app/src/server/service/page/index.ts +++ b/apps/app/src/server/service/page/index.ts @@ -1588,12 +1588,12 @@ class PageService implements IPageService { }) .pipe(batchStream) .on('error', () => { - readStream.destory(); + readStream.destroy(); writeStream.end(); }) .pipe(writeStream) .on('error', () => { - readStream.destory(); + readStream.destroy(); batchStream.destroy(); }); @@ -1932,12 +1932,12 @@ class PageService implements IPageService { }) .pipe(batchStream) .on('error', () => { - readStream.destory(); + readStream.destroy(); writeStream.end(); }) .pipe(writeStream) .on('error', () => { - readStream.destory(); + readStream.destroy(); batchStream.destroy(); }); @@ -2209,12 +2209,12 @@ class PageService implements IPageService { }) .pipe(batchStream) .on('error', () => { - readStream.destory(); + readStream.destroy(); writeStream.end(); }) .pipe(writeStream) .on('error', () => { - readStream.destory(); + readStream.destroy(); batchStream.destroy(); }); @@ -2639,7 +2639,7 @@ class PageService implements IPageService { }) .pipe(batchStream) .on('error', () => { - readStream.destory(); + readStream.destroy(); writeStream.end(); }) .pipe(writeStream) From 56bbb8aa939bcfc8ba88e83be4465b30cfed5424 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Fri, 8 Nov 2024 08:00:24 +0000 Subject: [PATCH 09/22] use pipeline --- .../services/growi-plugin/growi-plugin.ts | 4 +- .../features/openai/server/services/openai.ts | 17 +- ...to-page-id-schema-migration--fixed-7549.js | 34 +--- .../app/src/server/routes/apiv3/page/index.ts | 4 +- apps/app/src/server/service/export.js | 32 +--- .../src/server/service/file-uploader/local.ts | 12 +- .../src/server/service/growi-bridge/index.ts | 6 +- apps/app/src/server/service/import/import.ts | 48 +---- .../delete-completely-user-home-by-system.ts | 18 +- apps/app/src/server/service/page/index.ts | 179 ++---------------- .../service/search-delegator/elasticsearch.ts | 36 +--- 11 files changed, 49 insertions(+), 341 deletions(-) diff --git a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts index 6bdbc7cafa4..5b091c564ef 100644 --- a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts +++ b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts @@ -1,5 +1,6 @@ import fs, { readFileSync } from 'fs'; import path from 'path'; +import { pipeline } from 'stream'; import { GrowiPluginType } from '@growi/core'; import type { GrowiThemeMetadata, ViteManifest } from '@growi/core'; @@ -212,7 +213,8 @@ export class GrowiPluginService implements IGrowiPluginService { const readZipStream = fs.createReadStream(zipFilePath); const writeUnZipStream = unzipStream.Extract({ path: destPath.toString() }); - const unzipFileStream = readZipStream.on('error', () => { writeUnZipStream.end() }).pipe(writeUnZipStream).on('error', () => { readZipStream.destroy() }); + const unzipFileStream = pipeline(readZipStream, writeUnZipStream); + await streamToPromise(unzipFileStream); } diff --git a/apps/app/src/features/openai/server/services/openai.ts b/apps/app/src/features/openai/server/services/openai.ts index 1d1990d02d2..d812390e8d1 100644 --- a/apps/app/src/features/openai/server/services/openai.ts +++ b/apps/app/src/features/openai/server/services/openai.ts @@ -1,5 +1,6 @@ import assert from 'node:assert'; import { Readable, Transform } from 'stream'; +import { pipeline } from 'stream/promises'; import { PageGrant, isPopulated } from '@growi/core'; import type { HydratedDocument, Types } from 'mongoose'; @@ -342,21 +343,7 @@ class OpenaiService implements IOpenaiService { }, }); - pagesStream - .on('error', () => { - batchStrem.end(); - createVectorStoreFileStream.end(); - }) - .pipe(batchStrem) - .on('error', () => { - pagesStream.destroy(); - createVectorStoreFileStream.end(); - }) - .pipe(createVectorStoreFileStream) - .on('error', () => { - pagesStream.destroy(); - batchStrem.destroy(); - }); + await pipeline(pagesStream, batchStrem, createVectorStoreFileStream); } async rebuildVectorStore(page: HydratedDocument) { diff --git a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js index d1c158f6dbf..e82434462be 100644 --- a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js +++ b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js @@ -1,4 +1,4 @@ -import { Writable } from 'stream'; +import { pipeline, Writable } from 'stream'; import mongoose from 'mongoose'; import streamToPromise from 'stream-to-promise'; @@ -56,21 +56,7 @@ module.exports = { }, }); - pagesStream - .on('error', () => { - batchStrem.end(); - migratePagesStream.end(); - }) - .pipe(batchStrem) - .on('error', () => { - pagesStream.destroy(); - migratePagesStream.end(); - }) - .pipe(migratePagesStream) - .on('error', () => { - pagesStream.destroy(); - batchStrem.destroy(); - }); + pipeline(pagesStream, batchStrem, migratePagesStream); await streamToPromise(migratePagesStream); @@ -119,21 +105,7 @@ module.exports = { }, }); - pagesStream - .on('error', () => { - batchStrem.end(); - migratePagesStream.end(); - }) - .pipe(batchStrem) - .on('error', () => { - pagesStream.destroy(); - migratePagesStream.end(); - }) - .pipe(migratePagesStream) - .on('error', () => { - pagesStream.destroy(); - batchStrem.destroy(); - }); + pipeline(pagesStream, batchStrem, migratePagesStream); await streamToPromise(migratePagesStream); diff --git a/apps/app/src/server/routes/apiv3/page/index.ts b/apps/app/src/server/routes/apiv3/page/index.ts index 62ef10d9916..2961c513c69 100644 --- a/apps/app/src/server/routes/apiv3/page/index.ts +++ b/apps/app/src/server/routes/apiv3/page/index.ts @@ -1,5 +1,5 @@ import path from 'path'; -import type { Readable } from 'stream'; +import { pipeline, type Readable } from 'stream'; import type { IPage } from '@growi/core'; import { @@ -760,7 +760,7 @@ module.exports = (crowi) => { }; await crowi.activityService.createActivity(parameters); - return stream.pipe(res).on('error', () => { stream.destroy() }); + return pipeline(stream, res); }); /** diff --git a/apps/app/src/server/service/export.js b/apps/app/src/server/service/export.js index 77f0e7c96b4..a25347f8341 100644 --- a/apps/app/src/server/service/export.js +++ b/apps/app/src/server/service/export.js @@ -7,7 +7,7 @@ const logger = loggerFactory('growi:services:ExportService'); // eslint-disable- const fs = require('fs'); const path = require('path'); -const { Transform } = require('stream'); +const { Transform, pipeline } = require('stream'); const archiver = require('archiver'); const mongoose = require('mongoose'); @@ -196,30 +196,7 @@ class ExportService { const jsonFileToWrite = path.join(this.baseDir, `${collectionName}.json`); const writeStream = fs.createWriteStream(jsonFileToWrite, { encoding: this.growiBridgeService.getEncoding() }); - readStream - .on('error', () => { - logStream.end(); - transformStream.end(); - writeStream.end(); - }) - .pipe(logStream) - .on('error', () => { - readStream.destroy(); - transformStream.end(); - writeStream.end(); - }) - .pipe(transformStream) - .on('error', () => { - readStream.destroy(); - logStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - logStream.destroy(); - readStream.destroy(); - }); + pipeline(readStream, logStream, transformStream, writeStream); await streamToPromise(writeStream); @@ -375,10 +352,7 @@ class ExportService { const output = fs.createWriteStream(zipFile); // pipe archive data to the file - archive - .on('error', () => { output.end() }) - .pipe(output) - .on('error', () => { archive.destroy() }); + pipeline(archive, output); // finalize the archive (ie we are done appending files but streams have to finish yet) // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand diff --git a/apps/app/src/server/service/file-uploader/local.ts b/apps/app/src/server/service/file-uploader/local.ts index c23b5ae4eea..79956e23346 100644 --- a/apps/app/src/server/service/file-uploader/local.ts +++ b/apps/app/src/server/service/file-uploader/local.ts @@ -1,5 +1,5 @@ import type { ReadStream } from 'fs'; -import { Readable } from 'stream'; +import { pipeline, Readable } from 'stream'; import type { Response } from 'express'; @@ -165,10 +165,7 @@ module.exports = function(crowi) { const writeStream = fs.createWriteStream(filePath); - const stream = fileStream - .on('error', () => { writeStream.end() }) - .pipe(writeStream) - .on('error', () => { fileStream.destroy() }); + const stream = pipeline(fileStream, writeStream); return streamToPromise(stream); }; @@ -183,10 +180,7 @@ module.exports = function(crowi) { fileStream.push(data); fileStream.push(null); // EOF const writeStream = fs.createWriteStream(absFilePath); - const stream = fileStream - .on('error', () => { writeStream.end() }) - .pipe(writeStream) - .on('error', () => { fileStream.destroy() }); + const stream = pipeline(fileStream, writeStream); return streamToPromise(stream); }; diff --git a/apps/app/src/server/service/growi-bridge/index.ts b/apps/app/src/server/service/growi-bridge/index.ts index 80fce77cb17..8d6aca04a16 100644 --- a/apps/app/src/server/service/growi-bridge/index.ts +++ b/apps/app/src/server/service/growi-bridge/index.ts @@ -1,5 +1,6 @@ import fs from 'fs'; import path from 'path'; +import { pipeline } from 'stream'; import streamToPromise from 'stream-to-promise'; import unzipStream, { type Entry } from 'unzip-stream'; @@ -79,10 +80,7 @@ class GrowiBridgeService { const readStream = fs.createReadStream(zipFile); const parseStream = unzipStream.Parse(); - const unzipStreamPipe = readStream - .on('error', () => parseStream.end()) - .pipe(parseStream) - .on('error', () => readStream.destroy()); + const unzipStreamPipe = pipeline(readStream, parseStream); let tapPromise; diff --git a/apps/app/src/server/service/import/import.ts b/apps/app/src/server/service/import/import.ts index 5f548069fb1..b852035b68a 100644 --- a/apps/app/src/server/service/import/import.ts +++ b/apps/app/src/server/service/import/import.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; import type { EventEmitter } from 'stream'; -import { Writable, Transform } from 'stream'; +import { Writable, Transform, pipeline } from 'stream'; import JSONStream from 'JSONStream'; import gc from 'expose-gc/function'; @@ -267,41 +267,7 @@ export class ImportService { }, }); - readStream - .on('error', () => { - jsonStream.end(); - convertStream.end(); - batchStream.end(); - writeStream.end(); - }) - .pipe(jsonStream) - .on('error', () => { - readStream.destroy(); - convertStream.end(); - batchStream.end(); - writeStream.end(); - }) - .pipe(convertStream) - .on('error', () => { - readStream.destroy(); - jsonStream.destroy(); - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - jsonStream.destroy(); - convertStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - jsonStream.destroy(); - convertStream.destroy(); - batchStream.destroy(); - }); + pipeline(readStream, jsonStream, convertStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -380,10 +346,7 @@ export class ImportService { async unzip(zipFile) { const readStream = fs.createReadStream(zipFile); const parseStream = unzipStream.Parse(); - const unzipStreamPipe = readStream - .on('error', () => { parseStream.end() }) - .pipe(parseStream) - .on('error', () => { readStream.destroy() }); + const unzipStreamPipe = pipeline(readStream, parseStream); const files: string[] = []; unzipStreamPipe.on('entry', (/** @type {Entry} */ entry) => { @@ -404,10 +367,7 @@ export class ImportService { else { const jsonFile = path.join(this.baseDir, fileName); const writeStream = fs.createWriteStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() }); - entry - .on('error', () => { writeStream.end() }) - .pipe(writeStream) - .on('error', () => { entry.destroy() }); + pipeline(entry, writeStream); files.push(jsonFile); } }); diff --git a/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts b/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts index fcebae418c7..d0b0daad2e5 100644 --- a/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts +++ b/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts @@ -1,4 +1,4 @@ -import { Writable } from 'stream'; +import { pipeline, Writable } from 'stream'; import { getIdForRef } from '@growi/core'; import type { IPage, Ref } from '@growi/core'; @@ -110,21 +110,7 @@ export const deleteCompletelyUserHomeBySystem = async(userHomepagePath: string, }, }); - readStream - .on('error', () => { - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - }); + pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); // ────────┤ end │───────── diff --git a/apps/app/src/server/service/page/index.ts b/apps/app/src/server/service/page/index.ts index aeb59defa9f..9d2eef83e96 100644 --- a/apps/app/src/server/service/page/index.ts +++ b/apps/app/src/server/service/page/index.ts @@ -1,6 +1,6 @@ import type EventEmitter from 'events'; import pathlib from 'path'; -import { Readable, Writable } from 'stream'; +import { pipeline, Readable, Writable } from 'stream'; import { PageStatus, YDocStatus, getIdForRef, @@ -1045,21 +1045,8 @@ class PageService implements IPageService { }, }); - readStream - .on('error', () => { - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - }); + pipeline(readStream, batchStream, writeStream); + await streamToPromise(writeStream); } @@ -1097,21 +1084,8 @@ class PageService implements IPageService { }, }); - readStream - .on('error', () => { - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - }); + pipeline(readStream, batchStream, writeStream); + await streamToPromise(writeStream); } @@ -1527,21 +1501,7 @@ class PageService implements IPageService { }, }); - readStream - .on('error', () => { - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - }); + pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -1581,21 +1541,7 @@ class PageService implements IPageService { }, }); - readStream - .on('error', () => { - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - }); + pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -1925,21 +1871,7 @@ class PageService implements IPageService { }, }); - readStream - .on('error', () => { - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - }); + pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -2202,21 +2134,7 @@ class PageService implements IPageService { }, }); - readStream - .on('error', () => { - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - }); + pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -2503,21 +2421,8 @@ class PageService implements IPageService { }, }); - childPagesReadableStream - .on('error', () => { - batchStream.end(); - childPagesWritable.end(); - }) - .pipe(batchStream) - .on('error', () => { - childPagesReadableStream.destroy(); - childPagesWritable.end(); - }) - .pipe(childPagesWritable) - .on('error', () => { - childPagesReadableStream.destroy(); - childPagesWritable.destroy(); - }); + pipeline(childPagesReadableStream, batchStream, childPagesWritable); + await streamToPromise(childPagesWritable); } @@ -2584,21 +2489,7 @@ class PageService implements IPageService { }, }); - readStream - .on('error', () => { - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - }); + pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -2632,18 +2523,7 @@ class PageService implements IPageService { }, }); - readStream - .on('error', () => { - batchStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - writeStream.end(); - }) - .pipe(writeStream) - .on('error', () => {}); + pipeline(readStream, batchStream, writeStream); await streamToPromise(readStream); @@ -3491,21 +3371,7 @@ class PageService implements IPageService { }, }); - pagesStream - .on('error', () => { - batchStream.end(); - migratePagesStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - pagesStream.destroy(); - migratePagesStream.end(); - }) - .pipe(migratePagesStream) - .on('error', () => { - pagesStream.destroy(); - batchStream.destroy(); - }); + pipeline(pagesStream, batchStream, migratePagesStream); await streamToPromise(migratePagesStream); @@ -3622,21 +3488,8 @@ class PageService implements IPageService { callback(); }, }); - pageCursor - .on('error', () => { - batchStream.end(); - recountWriteStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - pageCursor.destroy(); - recountWriteStream.end(); - }) - .pipe(recountWriteStream) - .on('error', () => { - pageCursor.destroy(); - batchStream.destroy(); - }); + + pipeline(pageCursor, batchStream, recountWriteStream); await streamToPromise(recountWriteStream); } diff --git a/apps/app/src/server/service/search-delegator/elasticsearch.ts b/apps/app/src/server/service/search-delegator/elasticsearch.ts index 9eb5d39b7eb..57c7d628323 100644 --- a/apps/app/src/server/service/search-delegator/elasticsearch.ts +++ b/apps/app/src/server/service/search-delegator/elasticsearch.ts @@ -1,4 +1,4 @@ -import { Writable, Transform } from 'stream'; +import { Writable, Transform, pipeline } from 'stream'; import { URL } from 'url'; import { getIdStringForRef, type IPage } from '@growi/core'; @@ -553,32 +553,14 @@ class ElasticsearchDelegator implements SearchDelegator { - batchStream.end(); - appendTagNamesStream.end(); - writeStream.end(); - }) - .pipe(batchStream) - .on('error', () => { - readStream.destroy(); - appendTagNamesStream.end(); - writeStream.end(); - }) - .pipe(appendTagNamesStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - writeStream.end(); - }) - // .pipe(appendEmbeddingStream) - // .pipe(appendFileUploadedStream) - .pipe(writeStream) - .on('error', () => { - readStream.destroy(); - batchStream.destroy(); - appendTagNamesStream.destroy(); - }); + pipeline( + readStream, + batchStream, + appendTagNamesStream, + // appendEmbeddingStream, + // appendFileUploadedStream, + writeStream, + ); return streamToPromise(writeStream); } From 7642fad8469bedb49b3e2d93f32a8add3a94d540 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 04:44:35 +0000 Subject: [PATCH 10/22] use async pipeline --- ...to-page-id-schema-migration--fixed-7549.js | 7 +++--- apps/app/src/server/service/export.js | 7 +++--- apps/app/src/server/service/import/import.ts | 3 ++- .../delete-completely-user-home-by-system.ts | 5 ++-- apps/app/src/server/service/page/index.ts | 25 ++++++++++--------- .../service/search-delegator/elasticsearch.ts | 5 ++-- 6 files changed, 29 insertions(+), 23 deletions(-) diff --git a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js index e82434462be..e2d7b348fd0 100644 --- a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js +++ b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js @@ -1,4 +1,5 @@ -import { pipeline, Writable } from 'stream'; +import { Writable } from 'stream'; +import { pipeline } from 'stream/promises'; import mongoose from 'mongoose'; import streamToPromise from 'stream-to-promise'; @@ -56,7 +57,7 @@ module.exports = { }, }); - pipeline(pagesStream, batchStrem, migratePagesStream); + await pipeline(pagesStream, batchStrem, migratePagesStream); await streamToPromise(migratePagesStream); @@ -105,7 +106,7 @@ module.exports = { }, }); - pipeline(pagesStream, batchStrem, migratePagesStream); + await pipeline(pagesStream, batchStrem, migratePagesStream); await streamToPromise(migratePagesStream); diff --git a/apps/app/src/server/service/export.js b/apps/app/src/server/service/export.js index a25347f8341..31b218b9b9d 100644 --- a/apps/app/src/server/service/export.js +++ b/apps/app/src/server/service/export.js @@ -7,7 +7,8 @@ const logger = loggerFactory('growi:services:ExportService'); // eslint-disable- const fs = require('fs'); const path = require('path'); -const { Transform, pipeline } = require('stream'); +const { Transform } = require('stream'); +const { pipeline } = require('stream/promises'); const archiver = require('archiver'); const mongoose = require('mongoose'); @@ -196,7 +197,7 @@ class ExportService { const jsonFileToWrite = path.join(this.baseDir, `${collectionName}.json`); const writeStream = fs.createWriteStream(jsonFileToWrite, { encoding: this.growiBridgeService.getEncoding() }); - pipeline(readStream, logStream, transformStream, writeStream); + await pipeline(readStream, logStream, transformStream, writeStream); await streamToPromise(writeStream); @@ -352,7 +353,7 @@ class ExportService { const output = fs.createWriteStream(zipFile); // pipe archive data to the file - pipeline(archive, output); + await pipeline(archive, output); // finalize the archive (ie we are done appending files but streams have to finish yet) // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand diff --git a/apps/app/src/server/service/import/import.ts b/apps/app/src/server/service/import/import.ts index b852035b68a..20ded771814 100644 --- a/apps/app/src/server/service/import/import.ts +++ b/apps/app/src/server/service/import/import.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import path from 'path'; import type { EventEmitter } from 'stream'; import { Writable, Transform, pipeline } from 'stream'; +import { pipeline as pipelinePromise } from 'stream/promises'; import JSONStream from 'JSONStream'; import gc from 'expose-gc/function'; @@ -267,7 +268,7 @@ export class ImportService { }, }); - pipeline(readStream, jsonStream, convertStream, batchStream, writeStream); + await pipelinePromise(readStream, jsonStream, convertStream, batchStream, writeStream); await streamToPromise(writeStream); diff --git a/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts b/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts index d0b0daad2e5..b53f449176b 100644 --- a/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts +++ b/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts @@ -1,4 +1,5 @@ -import { pipeline, Writable } from 'stream'; +import { Writable } from 'stream'; +import { pipeline } from 'stream/promises'; import { getIdForRef } from '@growi/core'; import type { IPage, Ref } from '@growi/core'; @@ -110,7 +111,7 @@ export const deleteCompletelyUserHomeBySystem = async(userHomepagePath: string, }, }); - pipeline(readStream, batchStream, writeStream); + await pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); // ────────┤ end │───────── diff --git a/apps/app/src/server/service/page/index.ts b/apps/app/src/server/service/page/index.ts index 9d2eef83e96..8fe716cfae4 100644 --- a/apps/app/src/server/service/page/index.ts +++ b/apps/app/src/server/service/page/index.ts @@ -1,6 +1,7 @@ import type EventEmitter from 'events'; import pathlib from 'path'; -import { pipeline, Readable, Writable } from 'stream'; +import { Readable, Writable } from 'stream'; +import { pipeline } from 'stream/promises'; import { PageStatus, YDocStatus, getIdForRef, @@ -1045,7 +1046,7 @@ class PageService implements IPageService { }, }); - pipeline(readStream, batchStream, writeStream); + await pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); } @@ -1084,7 +1085,7 @@ class PageService implements IPageService { }, }); - pipeline(readStream, batchStream, writeStream); + await pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); } @@ -1501,7 +1502,7 @@ class PageService implements IPageService { }, }); - pipeline(readStream, batchStream, writeStream); + await pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -1541,7 +1542,7 @@ class PageService implements IPageService { }, }); - pipeline(readStream, batchStream, writeStream); + await pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -1871,7 +1872,7 @@ class PageService implements IPageService { }, }); - pipeline(readStream, batchStream, writeStream); + await pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -2134,7 +2135,7 @@ class PageService implements IPageService { }, }); - pipeline(readStream, batchStream, writeStream); + await pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -2421,7 +2422,7 @@ class PageService implements IPageService { }, }); - pipeline(childPagesReadableStream, batchStream, childPagesWritable); + await pipeline(childPagesReadableStream, batchStream, childPagesWritable); await streamToPromise(childPagesWritable); } @@ -2489,7 +2490,7 @@ class PageService implements IPageService { }, }); - pipeline(readStream, batchStream, writeStream); + await pipeline(readStream, batchStream, writeStream); await streamToPromise(writeStream); @@ -2523,7 +2524,7 @@ class PageService implements IPageService { }, }); - pipeline(readStream, batchStream, writeStream); + await pipeline(readStream, batchStream, writeStream); await streamToPromise(readStream); @@ -3371,7 +3372,7 @@ class PageService implements IPageService { }, }); - pipeline(pagesStream, batchStream, migratePagesStream); + await pipeline(pagesStream, batchStream, migratePagesStream); await streamToPromise(migratePagesStream); @@ -3489,7 +3490,7 @@ class PageService implements IPageService { }, }); - pipeline(pageCursor, batchStream, recountWriteStream); + await pipeline(pageCursor, batchStream, recountWriteStream); await streamToPromise(recountWriteStream); } diff --git a/apps/app/src/server/service/search-delegator/elasticsearch.ts b/apps/app/src/server/service/search-delegator/elasticsearch.ts index 57c7d628323..10aed5cb2fa 100644 --- a/apps/app/src/server/service/search-delegator/elasticsearch.ts +++ b/apps/app/src/server/service/search-delegator/elasticsearch.ts @@ -1,4 +1,5 @@ -import { Writable, Transform, pipeline } from 'stream'; +import { Writable, Transform } from 'stream'; +import { pipeline } from 'stream/promises'; import { URL } from 'url'; import { getIdStringForRef, type IPage } from '@growi/core'; @@ -553,7 +554,7 @@ class ElasticsearchDelegator implements SearchDelegator Date: Mon, 18 Nov 2024 06:09:19 +0000 Subject: [PATCH 11/22] fix code --- .../server/services/growi-plugin/growi-plugin.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts index 5b091c564ef..89c6439a870 100644 --- a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts +++ b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts @@ -211,10 +211,7 @@ export class GrowiPluginService implements IGrowiPluginService { private async unzip(zipFilePath: fs.PathLike, destPath: fs.PathLike): Promise { try { const readZipStream = fs.createReadStream(zipFilePath); - const writeUnZipStream = unzipStream.Extract({ path: destPath.toString() }); - - const unzipFileStream = pipeline(readZipStream, writeUnZipStream); - + const unzipFileStream = pipeline(readZipStream, unzipStream.Extract({ path: destPath.toString() })); await streamToPromise(unzipFileStream); } From 31b0c5155f68bf5876f7ba53349b0618b5bf1bac Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 06:17:32 +0000 Subject: [PATCH 12/22] use async --- .../growi-plugin/server/services/growi-plugin/growi-plugin.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts index 89c6439a870..99c70c443fa 100644 --- a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts +++ b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts @@ -1,6 +1,6 @@ import fs, { readFileSync } from 'fs'; import path from 'path'; -import { pipeline } from 'stream'; +import { pipeline } from 'stream/promises'; import { GrowiPluginType } from '@growi/core'; import type { GrowiThemeMetadata, ViteManifest } from '@growi/core'; @@ -211,7 +211,7 @@ export class GrowiPluginService implements IGrowiPluginService { private async unzip(zipFilePath: fs.PathLike, destPath: fs.PathLike): Promise { try { const readZipStream = fs.createReadStream(zipFilePath); - const unzipFileStream = pipeline(readZipStream, unzipStream.Extract({ path: destPath.toString() })); + const unzipFileStream = await pipeline(readZipStream, unzipStream.Extract({ path: destPath.toString() })); await streamToPromise(unzipFileStream); } From 2a6e94353185a5dd5271ae59afa7278b558a4f76 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 06:45:23 +0000 Subject: [PATCH 13/22] update plugin repo name --- .../server/services/growi-plugin/growi-plugin.integ.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.integ.ts b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.integ.ts index 7acd62714c1..76b9aa55aee 100644 --- a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.integ.ts +++ b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.integ.ts @@ -11,7 +11,7 @@ describe('Installing a GROWI template plugin', () => { it('install() should success', async() => { // when const result = await growiPluginService.install({ - url: 'https://github.com/weseek/growi-plugin-templates-for-office', + url: 'https://github.com/growilabs/growi-plugin-templates-for-office', }); const count = await GrowiPlugin.count({ 'meta.name': 'growi-plugin-templates-for-office' }); @@ -42,7 +42,7 @@ describe('Installing a GROWI template plugin', () => { // when const result = await growiPluginService.install({ - url: 'https://github.com/weseek/growi-plugin-templates-for-office', + url: 'https://github.com/growilabs/growi-plugin-templates-for-office', }); const count2 = await GrowiPlugin.count({ 'meta.name': 'growi-plugin-templates-for-office' }); @@ -59,7 +59,7 @@ describe('Installing a GROWI theme plugin', () => { it('install() should success', async() => { // when const result = await growiPluginService.install({ - url: 'https://github.com/weseek/growi-plugin-theme-vivid-internet', + url: 'https://github.com/growilabs/growi-plugin-theme-vivid-internet', }); const count = await GrowiPlugin.count({ 'meta.name': 'growi-plugin-theme-vivid-internet' }); From 0641e887f320255dcfa9f49765080397676531eb Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 06:46:28 +0000 Subject: [PATCH 14/22] clean code --- .../server/services/growi-plugin/growi-plugin.integ.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.integ.ts b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.integ.ts index 76b9aa55aee..6ac314b926a 100644 --- a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.integ.ts +++ b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.integ.ts @@ -20,7 +20,7 @@ describe('Installing a GROWI template plugin', () => { expect(count).toBe(1); expect(fs.existsSync(path.join( PLUGIN_STORING_PATH, - 'weseek', + 'growilabs', 'growi-plugin-templates-for-office', ))).toBeTruthy(); }); @@ -33,7 +33,7 @@ describe('Installing a GROWI template plugin', () => { // setup const dummyFilePath = path.join( PLUGIN_STORING_PATH, - 'weseek', + 'growilabs', 'growi-plugin-templates-for-office', 'dummy.txt', ); @@ -68,7 +68,7 @@ describe('Installing a GROWI theme plugin', () => { expect(count).toBe(1); expect(fs.existsSync(path.join( PLUGIN_STORING_PATH, - 'weseek', + 'growilabs', 'growi-plugin-theme-vivid-internet', ))).toBeTruthy(); }); @@ -88,7 +88,7 @@ describe('Installing a GROWI theme plugin', () => { expect(results.themeMetadata).not.toBeNull(); expect(results.themeHref).not.toBeNull(); expect(results.themeHref - .startsWith('/static/plugins/weseek/growi-plugin-theme-vivid-internet/dist/assets/style-')).toBeTruthy(); + .startsWith('/static/plugins/growilabs/growi-plugin-theme-vivid-internet/dist/assets/style-')).toBeTruthy(); }); }); From 02c563a9b1224c143e369caebe8fbf3ec60ea2b2 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 06:57:27 +0000 Subject: [PATCH 15/22] fix code --- .../growi-plugin/server/services/growi-plugin/growi-plugin.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts index 99c70c443fa..89c6439a870 100644 --- a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts +++ b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts @@ -1,6 +1,6 @@ import fs, { readFileSync } from 'fs'; import path from 'path'; -import { pipeline } from 'stream/promises'; +import { pipeline } from 'stream'; import { GrowiPluginType } from '@growi/core'; import type { GrowiThemeMetadata, ViteManifest } from '@growi/core'; @@ -211,7 +211,7 @@ export class GrowiPluginService implements IGrowiPluginService { private async unzip(zipFilePath: fs.PathLike, destPath: fs.PathLike): Promise { try { const readZipStream = fs.createReadStream(zipFilePath); - const unzipFileStream = await pipeline(readZipStream, unzipStream.Extract({ path: destPath.toString() })); + const unzipFileStream = pipeline(readZipStream, unzipStream.Extract({ path: destPath.toString() })); await streamToPromise(unzipFileStream); } From fcdfeecca15f17e5864f06d703808166f8723ec9 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 07:05:21 +0000 Subject: [PATCH 16/22] dont use stream-to-promise --- .../server/services/growi-plugin/growi-plugin.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts index 89c6439a870..582fb4616dd 100644 --- a/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts +++ b/apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts @@ -1,6 +1,6 @@ import fs, { readFileSync } from 'fs'; import path from 'path'; -import { pipeline } from 'stream'; +import { pipeline } from 'stream/promises'; import { GrowiPluginType } from '@growi/core'; import type { GrowiThemeMetadata, ViteManifest } from '@growi/core'; @@ -9,7 +9,6 @@ import { importPackageJson, validateGrowiDirective } from '@growi/pluginkit/dist // eslint-disable-next-line no-restricted-imports import axios from 'axios'; import type mongoose from 'mongoose'; -import streamToPromise from 'stream-to-promise'; import unzipStream from 'unzip-stream'; import loggerFactory from '~/utils/logger'; @@ -211,9 +210,7 @@ export class GrowiPluginService implements IGrowiPluginService { private async unzip(zipFilePath: fs.PathLike, destPath: fs.PathLike): Promise { try { const readZipStream = fs.createReadStream(zipFilePath); - const unzipFileStream = pipeline(readZipStream, unzipStream.Extract({ path: destPath.toString() })); - - await streamToPromise(unzipFileStream); + await pipeline(readZipStream, unzipStream.Extract({ path: destPath.toString() })); } catch (err) { logger.error(err); From b78e1999d0f4e4dea95b915bc0d0d14a24e1ee90 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 08:18:39 +0000 Subject: [PATCH 17/22] use async pipeline and remove stream-to-promise --- ...to-page-id-schema-migration--fixed-7549.js | 4 ---- apps/app/src/server/service/export.js | 7 ++---- .../src/server/service/file-uploader/local.ts | 10 ++++----- .../src/server/service/growi-bridge/index.ts | 5 ++--- apps/app/src/server/service/import/import.ts | 7 ++---- .../delete-completely-user-home-by-system.ts | 3 --- apps/app/src/server/service/page/index.ts | 22 ------------------- .../service/search-delegator/elasticsearch.ts | 6 ++--- 8 files changed, 12 insertions(+), 52 deletions(-) diff --git a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js index e2d7b348fd0..0f07eac35d1 100644 --- a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js +++ b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js @@ -59,8 +59,6 @@ module.exports = { await pipeline(pagesStream, batchStrem, migratePagesStream); - await streamToPromise(migratePagesStream); - logger.info('Migration has successfully applied'); }, @@ -108,8 +106,6 @@ module.exports = { await pipeline(pagesStream, batchStrem, migratePagesStream); - await streamToPromise(migratePagesStream); - logger.info('Migration down has successfully applied'); }, }; diff --git a/apps/app/src/server/service/export.js b/apps/app/src/server/service/export.js index 31b218b9b9d..c9ef04045a3 100644 --- a/apps/app/src/server/service/export.js +++ b/apps/app/src/server/service/export.js @@ -199,8 +199,6 @@ class ExportService { await pipeline(readStream, logStream, transformStream, writeStream); - await streamToPromise(writeStream); - return writeStream.path; } @@ -353,13 +351,12 @@ class ExportService { const output = fs.createWriteStream(zipFile); // pipe archive data to the file - await pipeline(archive, output); + const stream = pipeline(archive, output); // finalize the archive (ie we are done appending files but streams have to finish yet) // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand archive.finalize(); - - await streamToPromise(archive); + await stream; logger.info(`zipped GROWI data into ${zipFile} (${archive.pointer()} bytes)`); diff --git a/apps/app/src/server/service/file-uploader/local.ts b/apps/app/src/server/service/file-uploader/local.ts index 79956e23346..1c28fe154ce 100644 --- a/apps/app/src/server/service/file-uploader/local.ts +++ b/apps/app/src/server/service/file-uploader/local.ts @@ -1,5 +1,6 @@ import type { ReadStream } from 'fs'; -import { pipeline, Readable } from 'stream'; +import { Readable } from 'stream'; +import { pipeline } from 'stream/promises'; import type { Response } from 'express'; @@ -24,7 +25,6 @@ const fsPromises = require('fs/promises'); const path = require('path'); const mkdir = require('mkdirp'); -const streamToPromise = require('stream-to-promise'); const urljoin = require('url-join'); @@ -165,8 +165,7 @@ module.exports = function(crowi) { const writeStream = fs.createWriteStream(filePath); - const stream = pipeline(fileStream, writeStream); - return streamToPromise(stream); + return pipeline(fileStream, writeStream); }; lib.saveFile = async function({ filePath, contentType, data }) { @@ -180,8 +179,7 @@ module.exports = function(crowi) { fileStream.push(data); fileStream.push(null); // EOF const writeStream = fs.createWriteStream(absFilePath); - const stream = pipeline(fileStream, writeStream); - return streamToPromise(stream); + return pipeline(fileStream, writeStream); }; /** diff --git a/apps/app/src/server/service/growi-bridge/index.ts b/apps/app/src/server/service/growi-bridge/index.ts index 8d6aca04a16..61468286ecb 100644 --- a/apps/app/src/server/service/growi-bridge/index.ts +++ b/apps/app/src/server/service/growi-bridge/index.ts @@ -1,8 +1,7 @@ import fs from 'fs'; import path from 'path'; -import { pipeline } from 'stream'; +import { pipeline } from 'stream/promises'; -import streamToPromise from 'stream-to-promise'; import unzipStream, { type Entry } from 'unzip-stream'; import loggerFactory from '~/utils/logger'; @@ -103,7 +102,7 @@ class GrowiBridgeService { }); try { - await streamToPromise(unzipEntryStream); + await unzipEntryStream; await tapPromise; } // if zip is broken diff --git a/apps/app/src/server/service/import/import.ts b/apps/app/src/server/service/import/import.ts index 20ded771814..fe078dedc2e 100644 --- a/apps/app/src/server/service/import/import.ts +++ b/apps/app/src/server/service/import/import.ts @@ -11,7 +11,6 @@ import type { } from 'mongodb'; import type { Document } from 'mongoose'; import mongoose from 'mongoose'; -import streamToPromise from 'stream-to-promise'; import unzipStream from 'unzip-stream'; import { ImportMode } from '~/models/admin/import-mode'; @@ -270,8 +269,6 @@ export class ImportService { await pipelinePromise(readStream, jsonStream, convertStream, batchStream, writeStream); - await streamToPromise(writeStream); - // clean up tmp directory fs.unlinkSync(jsonFile); } @@ -347,7 +344,7 @@ export class ImportService { async unzip(zipFile) { const readStream = fs.createReadStream(zipFile); const parseStream = unzipStream.Parse(); - const unzipStreamPipe = pipeline(readStream, parseStream); + const unzipStreamPipe = pipelinePromise(readStream, parseStream); const files: string[] = []; unzipStreamPipe.on('entry', (/** @type {Entry} */ entry) => { @@ -373,7 +370,7 @@ export class ImportService { } }); - await streamToPromise(unzipStreamPipe); + await unzipStreamPipe; return files; } diff --git a/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts b/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts index b53f449176b..efc84cb8c51 100644 --- a/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts +++ b/apps/app/src/server/service/page/delete-completely-user-home-by-system.ts @@ -6,7 +6,6 @@ import type { IPage, Ref } from '@growi/core'; import { isUsersHomepage } from '@growi/core/dist/utils/page-path-utils'; import type { HydratedDocument } from 'mongoose'; import mongoose from 'mongoose'; -import streamToPromise from 'stream-to-promise'; import type { PageDocument, PageModel } from '~/server/models/page'; import { createBatchStream } from '~/server/util/batch-stream'; @@ -112,8 +111,6 @@ export const deleteCompletelyUserHomeBySystem = async(userHomepagePath: string, }); await pipeline(readStream, batchStream, writeStream); - - await streamToPromise(writeStream); // ────────┤ end │───────── } catch (err) { diff --git a/apps/app/src/server/service/page/index.ts b/apps/app/src/server/service/page/index.ts index 1c9361f0699..fd639462538 100644 --- a/apps/app/src/server/service/page/index.ts +++ b/apps/app/src/server/service/page/index.ts @@ -19,7 +19,6 @@ import { import escapeStringRegexp from 'escape-string-regexp'; import type { Cursor, HydratedDocument } from 'mongoose'; import mongoose from 'mongoose'; -import streamToPromise from 'stream-to-promise'; import { Comment } from '~/features/comment/server'; import type { ExternalUserGroupDocument } from '~/features/external-user-group/server/models/external-user-group'; @@ -1047,8 +1046,6 @@ class PageService implements IPageService { }); await pipeline(readStream, batchStream, writeStream); - - await streamToPromise(writeStream); } private async renameDescendantsWithStreamV4(targetPage, newPagePath, user, options = {}) { @@ -1086,8 +1083,6 @@ class PageService implements IPageService { }); await pipeline(readStream, batchStream, writeStream); - - await streamToPromise(writeStream); } /* @@ -1512,8 +1507,6 @@ class PageService implements IPageService { await pipeline(readStream, batchStream, writeStream); - await streamToPromise(writeStream); - return nNonEmptyDuplicatedPages; } @@ -1552,8 +1545,6 @@ class PageService implements IPageService { await pipeline(readStream, batchStream, writeStream); - await streamToPromise(writeStream); - return count; } @@ -1882,8 +1873,6 @@ class PageService implements IPageService { await pipeline(readStream, batchStream, writeStream); - await streamToPromise(writeStream); - return nDeletedNonEmptyPages; } @@ -2149,8 +2138,6 @@ class PageService implements IPageService { await pipeline(readStream, batchStream, writeStream); - await streamToPromise(writeStream); - return nDeletedNonEmptyPages; } @@ -2436,7 +2423,6 @@ class PageService implements IPageService { await pipeline(childPagesReadableStream, batchStream, childPagesWritable); - await streamToPromise(childPagesWritable); } async updateChildPagesGrant( @@ -2504,8 +2490,6 @@ class PageService implements IPageService { await pipeline(readStream, batchStream, writeStream); - await streamToPromise(writeStream); - return count; } @@ -2538,8 +2522,6 @@ class PageService implements IPageService { await pipeline(readStream, batchStream, writeStream); - await streamToPromise(readStream); - return count; } @@ -3386,8 +3368,6 @@ class PageService implements IPageService { await pipeline(pagesStream, batchStream, migratePagesStream); - await streamToPromise(migratePagesStream); - if (await Page.exists(matchFilter) && shouldContinue) { return this._normalizeParentRecursively( pathOrRegExps, @@ -3503,8 +3483,6 @@ class PageService implements IPageService { }); await pipeline(pageCursor, batchStream, recountWriteStream); - - await streamToPromise(recountWriteStream); } // update descendantCount of all pages that are ancestors of a provided pageId by count diff --git a/apps/app/src/server/service/search-delegator/elasticsearch.ts b/apps/app/src/server/service/search-delegator/elasticsearch.ts index 10aed5cb2fa..ed1fb03fd43 100644 --- a/apps/app/src/server/service/search-delegator/elasticsearch.ts +++ b/apps/app/src/server/service/search-delegator/elasticsearch.ts @@ -5,7 +5,6 @@ import { URL } from 'url'; import { getIdStringForRef, type IPage } from '@growi/core'; import gc from 'expose-gc/function'; import mongoose from 'mongoose'; -import streamToPromise from 'stream-to-promise'; import { SearchDelegatorName } from '~/interfaces/named-query'; import type { ISearchResult, ISearchResultData } from '~/interfaces/search'; @@ -554,7 +553,8 @@ class ElasticsearchDelegator implements SearchDelegator Date: Mon, 18 Nov 2024 08:37:12 +0000 Subject: [PATCH 18/22] clean code --- ...05-revision-path-to-page-id-schema-migration--fixed-7549.js | 1 - apps/app/src/server/service/export.js | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js index 0f07eac35d1..0fd07cea65c 100644 --- a/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js +++ b/apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js @@ -2,7 +2,6 @@ import { Writable } from 'stream'; import { pipeline } from 'stream/promises'; import mongoose from 'mongoose'; -import streamToPromise from 'stream-to-promise'; import getPageModel from '~/server/models/page'; import { Revision } from '~/server/models/revision'; diff --git a/apps/app/src/server/service/export.js b/apps/app/src/server/service/export.js index c9ef04045a3..7b7831feab1 100644 --- a/apps/app/src/server/service/export.js +++ b/apps/app/src/server/service/export.js @@ -12,7 +12,6 @@ const { pipeline } = require('stream/promises'); const archiver = require('archiver'); const mongoose = require('mongoose'); -const streamToPromise = require('stream-to-promise'); const CollectionProgressingStatus = require('../models/vo/collection-progressing-status'); @@ -108,7 +107,7 @@ class ExportService { writeStream.write(JSON.stringify(metaData)); writeStream.close(); - await streamToPromise(writeStream); + await pipeline(writeStream); return metaJson; } From 0b89da0f17f8d6e2e53b6f3ddfea92a37a9d8e01 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 08:37:19 +0000 Subject: [PATCH 19/22] remove stream-to-promise --- apps/app/package.json | 1 - pnpm-lock.yaml | 92 ++++++++++++++++++------------------------- 2 files changed, 38 insertions(+), 55 deletions(-) diff --git a/apps/app/package.json b/apps/app/package.json index 7f9d41b1ffc..54e2ee41e18 100644 --- a/apps/app/package.json +++ b/apps/app/package.json @@ -211,7 +211,6 @@ "remark-stringify": "^11.0.0", "sanitize-filename": "^1.6.3", "socket.io": "^4.7.5", - "stream-to-promise": "^3.0.0", "string-width": "=4.2.2", "superjson": "^1.9.1", "swagger-jsdoc": "^6.2.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 86dded67d12..77af8f0cea0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -639,9 +639,6 @@ importers: socket.io: specifier: ^4.7.5 version: 4.8.1 - stream-to-promise: - specifier: ^3.0.0 - version: 3.0.0 string-width: specifier: '=4.2.2' version: 4.2.2 @@ -10816,13 +10813,6 @@ packages: engines: {node: '>= 0.10.0'} hasBin: true - stream-to-array@2.3.0: - resolution: {integrity: sha512-UsZtOYEn4tWU2RGLOXr/o/xjRBftZRlG3dEWoaHr8j4GuypJ3isitGbVyjQKAuMu+xbiop8q224TjiZWc4XTZA==} - - stream-to-promise@3.0.0: - resolution: {integrity: sha512-h+7wLeFiYegOdgTfTxjRsrT7/Op7grnKEIHWgaO1RTHwcwk7xRreMr3S8XpDfDMesSxzgM2V4CxNCFAGo6ssnA==} - engines: {node: '>= 10'} - stream-transform@2.1.3: resolution: {integrity: sha512-9GHUiM5hMiCi6Y03jD2ARC1ettBXkQBoQAe7nJsPknnI0ow10aXjTnew8QtYQmLjzn974BnmWEAJgCY6ZP1DeQ==} @@ -13399,7 +13389,7 @@ snapshots: '@babel/traverse': 7.24.6 '@babel/types': 7.25.6 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13590,7 +13580,7 @@ snapshots: '@babel/helper-split-export-declaration': 7.24.6 '@babel/parser': 7.25.6 '@babel/types': 7.25.6 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -14095,7 +14085,7 @@ snapshots: '@elastic/elasticsearch@7.17.13': dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 hpagent: 0.1.2 ms: 2.1.3 secure-json-parse: 2.7.0 @@ -14111,7 +14101,7 @@ snapshots: '@elastic/transport@8.6.1': dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 hpagent: 1.2.0 ms: 2.1.3 secure-json-parse: 2.7.0 @@ -14222,7 +14212,7 @@ snapshots: '@eslint/eslintrc@2.0.3': dependencies: ajv: 6.12.6 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -14295,7 +14285,7 @@ snapshots: '@humanwhocodes/config-array@0.11.8': dependencies: '@humanwhocodes/object-schema': 1.2.1 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -14311,7 +14301,7 @@ snapshots: '@antfu/install-pkg': 0.4.1 '@antfu/utils': 0.7.10 '@iconify/types': 2.0.0 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 kolorist: 1.8.0 local-pkg: 0.5.0 mlly: 1.7.1 @@ -15770,7 +15760,7 @@ snapshots: '@swc-node/sourcemap-support': 0.5.0 '@swc/core': 1.5.25(@swc/helpers@0.5.11) colorette: 2.0.20 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 pirates: 4.0.6 tslib: 2.8.0 typescript: 5.4.2 @@ -16371,7 +16361,7 @@ snapshots: '@typescript-eslint/scope-manager': 5.59.7 '@typescript-eslint/type-utils': 5.59.7(eslint@8.41.0)(typescript@5.4.2) '@typescript-eslint/utils': 5.59.7(eslint@8.41.0)(typescript@5.4.2) - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 eslint: 8.41.0 grapheme-splitter: 1.0.4 ignore: 5.3.1 @@ -16401,7 +16391,7 @@ snapshots: '@typescript-eslint/scope-manager': 5.59.7 '@typescript-eslint/types': 5.59.7 '@typescript-eslint/typescript-estree': 5.59.7(typescript@5.4.2) - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 eslint: 8.41.0 optionalDependencies: typescript: 5.4.2 @@ -16430,7 +16420,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 5.59.7(typescript@5.4.2) '@typescript-eslint/utils': 5.59.7(eslint@8.41.0)(typescript@5.4.2) - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 eslint: 8.41.0 tsutils: 3.21.0(typescript@5.4.2) optionalDependencies: @@ -16459,7 +16449,7 @@ snapshots: dependencies: '@typescript-eslint/types': 5.59.7 '@typescript-eslint/visitor-keys': 5.59.7 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 globby: 11.1.0 is-glob: 4.0.3 semver: 7.6.3 @@ -16845,13 +16835,13 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color agent-base@7.1.1: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -17925,7 +17915,7 @@ snapshots: connect-mongo@4.6.0(express-session@1.18.0)(mongodb@4.17.2(@aws-sdk/client-sso-oidc@3.600.0)): dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 express-session: 1.18.0 kruptein: 3.0.6 mongodb: 4.17.2(@aws-sdk/client-sso-oidc@3.600.0) @@ -18405,6 +18395,10 @@ snapshots: dependencies: ms: 2.1.3 + debug@4.3.7: + dependencies: + ms: 2.1.3 + debug@4.3.7(supports-color@5.5.0): dependencies: ms: 2.1.3 @@ -18699,7 +18693,7 @@ snapshots: engine.io-client@6.6.2: dependencies: '@socket.io/component-emitter': 3.1.2 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 engine.io-parser: 5.2.3 ws: 8.17.1 xmlhttprequest-ssl: 2.1.2 @@ -18719,7 +18713,7 @@ snapshots: base64id: 2.0.0 cookie: 0.7.2 cors: 2.8.5 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 engine.io-parser: 5.2.3 ws: 8.17.1 transitivePeerDependencies: @@ -19165,7 +19159,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.0 @@ -19505,7 +19499,7 @@ snapshots: follow-redirects@1.15.9(debug@4.3.7): optionalDependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 follow-redirects@1.5.10: dependencies: @@ -20153,14 +20147,14 @@ snapshots: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.1 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -20183,14 +20177,14 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color https-proxy-agent@7.0.5: dependencies: agent-base: 7.1.1 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -20549,7 +20543,7 @@ snapshots: istanbul-lib-source-maps@4.0.1: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: @@ -21948,7 +21942,7 @@ snapshots: micromark@4.0.0: dependencies: '@types/debug': 4.1.7 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 decode-named-character-reference: 1.0.2 devlop: 1.1.0 micromark-core-commonmark: 2.0.1 @@ -22111,7 +22105,7 @@ snapshots: dependencies: async-mutex: 0.4.1 camelcase: 6.3.0 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 find-cache-dir: 3.3.2 follow-redirects: 1.15.9(debug@4.3.7) https-proxy-agent: 7.0.5 @@ -22219,7 +22213,7 @@ snapshots: mquery@4.0.3: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -22309,7 +22303,7 @@ snapshots: new-find-package-json@2.0.0: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -22840,7 +22834,7 @@ snapshots: passport-saml@3.2.4: dependencies: '@xmldom/xmldom': 0.7.13 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 passport-strategy: 1.0.0 xml-crypto: 2.1.5 xml-encryption: 2.0.0 @@ -23827,7 +23821,7 @@ snapshots: retry-request@4.2.2: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 extend: 3.0.2 transitivePeerDependencies: - supports-color @@ -24217,7 +24211,7 @@ snapshots: socket.io-adapter@2.5.5: dependencies: - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 ws: 8.17.1 transitivePeerDependencies: - bufferutil @@ -24227,7 +24221,7 @@ snapshots: socket.io-client@4.8.1: dependencies: '@socket.io/component-emitter': 3.1.2 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 engine.io-client: 6.6.2 socket.io-parser: 4.2.4 transitivePeerDependencies: @@ -24238,7 +24232,7 @@ snapshots: socket.io-parser@4.2.4: dependencies: '@socket.io/component-emitter': 3.1.2 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -24247,7 +24241,7 @@ snapshots: accepts: 1.3.8 base64id: 2.0.0 cors: 2.8.5 - debug: 4.3.7(supports-color@5.5.0) + debug: 4.3.7 engine.io: 6.6.2 socket.io-adapter: 2.5.5 socket.io-parser: 4.2.4 @@ -24392,16 +24386,6 @@ snapshots: commander: 2.20.3 limiter: 1.1.5 - stream-to-array@2.3.0: - dependencies: - any-promise: 1.3.0 - - stream-to-promise@3.0.0: - dependencies: - any-promise: 1.3.0 - end-of-stream: 1.4.4 - stream-to-array: 2.3.0 - stream-transform@2.1.3: dependencies: mixme: 0.5.10 From 3b4c6e04db6d51406606fcef492f05ce1f0b831c Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 08:49:42 +0000 Subject: [PATCH 20/22] remvoe unused val --- apps/app/src/server/service/growi-bridge/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/app/src/server/service/growi-bridge/index.ts b/apps/app/src/server/service/growi-bridge/index.ts index 61468286ecb..d2fdfc94621 100644 --- a/apps/app/src/server/service/growi-bridge/index.ts +++ b/apps/app/src/server/service/growi-bridge/index.ts @@ -79,11 +79,11 @@ class GrowiBridgeService { const readStream = fs.createReadStream(zipFile); const parseStream = unzipStream.Parse(); - const unzipStreamPipe = pipeline(readStream, parseStream); + const unzipEntryStream = pipeline(readStream, parseStream); let tapPromise; - const unzipEntryStream = unzipStreamPipe.on('entry', (entry: Entry) => { + unzipEntryStream.on('entry', (entry: Entry) => { const fileName = entry.path; const size = entry.size; // might be undefined in some archives if (fileName === this.getMetaFileName()) { From 075ab85d071fae36436d332aa0aecb1bbcdb2ec8 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 09:12:36 +0000 Subject: [PATCH 21/22] add types --- apps/app/src/server/service/file-uploader/local.ts | 5 +++-- apps/app/src/server/service/growi-bridge/index.ts | 5 +++-- apps/app/src/server/service/import/import.ts | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/app/src/server/service/file-uploader/local.ts b/apps/app/src/server/service/file-uploader/local.ts index 1c28fe154ce..e5ec39f41a4 100644 --- a/apps/app/src/server/service/file-uploader/local.ts +++ b/apps/app/src/server/service/file-uploader/local.ts @@ -1,4 +1,5 @@ import type { ReadStream } from 'fs'; +import type { Writable } from 'stream'; import { Readable } from 'stream'; import { pipeline } from 'stream/promises'; @@ -163,7 +164,7 @@ module.exports = function(crowi) { // mkdir -p mkdir.sync(dirpath); - const writeStream = fs.createWriteStream(filePath); + const writeStream: Writable = fs.createWriteStream(filePath); return pipeline(fileStream, writeStream); }; @@ -178,7 +179,7 @@ module.exports = function(crowi) { const fileStream = new Readable(); fileStream.push(data); fileStream.push(null); // EOF - const writeStream = fs.createWriteStream(absFilePath); + const writeStream: Writable = fs.createWriteStream(absFilePath); return pipeline(fileStream, writeStream); }; diff --git a/apps/app/src/server/service/growi-bridge/index.ts b/apps/app/src/server/service/growi-bridge/index.ts index d2fdfc94621..64c8558025e 100644 --- a/apps/app/src/server/service/growi-bridge/index.ts +++ b/apps/app/src/server/service/growi-bridge/index.ts @@ -1,6 +1,7 @@ import fs from 'fs'; import path from 'path'; -import { pipeline } from 'stream/promises'; +import { pipeline } from 'stream'; +import { pipeline as pipelinePromise } from 'stream/promises'; import unzipStream, { type Entry } from 'unzip-stream'; @@ -102,7 +103,7 @@ class GrowiBridgeService { }); try { - await unzipEntryStream; + await pipelinePromise([unzipEntryStream]); await tapPromise; } // if zip is broken diff --git a/apps/app/src/server/service/import/import.ts b/apps/app/src/server/service/import/import.ts index fe078dedc2e..4c47d963b91 100644 --- a/apps/app/src/server/service/import/import.ts +++ b/apps/app/src/server/service/import/import.ts @@ -344,10 +344,10 @@ export class ImportService { async unzip(zipFile) { const readStream = fs.createReadStream(zipFile); const parseStream = unzipStream.Parse(); - const unzipStreamPipe = pipelinePromise(readStream, parseStream); + const unzipStreamPipe = pipeline(readStream, parseStream); const files: string[] = []; - unzipStreamPipe.on('entry', (/** @type {Entry} */ entry) => { + const unzipEntryStream = unzipStreamPipe.on('entry', (/** @type {Entry} */ entry) => { const fileName = entry.path; // https://regex101.com/r/mD4eZs/6 // prevent from unexpecting attack doing unzip file (path traversal attack) @@ -370,7 +370,7 @@ export class ImportService { } }); - await unzipStreamPipe; + await pipelinePromise([unzipEntryStream]); return files; } From 40beb9c9816dc7ad93d505ab334a5abf18511a28 Mon Sep 17 00:00:00 2001 From: reiji-h Date: Mon, 18 Nov 2024 09:15:59 +0000 Subject: [PATCH 22/22] clean code --- apps/app/src/server/service/export.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/app/src/server/service/export.js b/apps/app/src/server/service/export.js index 7b7831feab1..6116dcd732e 100644 --- a/apps/app/src/server/service/export.js +++ b/apps/app/src/server/service/export.js @@ -107,7 +107,7 @@ class ExportService { writeStream.write(JSON.stringify(metaData)); writeStream.close(); - await pipeline(writeStream); + await pipeline([writeStream]); return metaJson; }