From 735ea296a431747ff589e5efc95f6f60b3e1acba Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 27 Jul 2023 07:23:34 +0800 Subject: [PATCH 1/7] perf: mediainfo router should use async/await to get file info --- package-lock.json | 8 +++---- package.json | 2 +- server/routes/api/torrents.ts | 15 ++++++++++--- server/util/async.ts | 8 +++++++ server/util/fileUtil.ts | 41 +++++++++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 server/util/async.ts diff --git a/package-lock.json b/package-lock.json index d1fdf2bed..cb04454f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,7 +52,7 @@ "@types/jsonwebtoken": "^9.0.2", "@types/lodash": "^4.14.195", "@types/morgan": "^1.9.4", - "@types/node": "^12.20.55", + "@types/node": "^18.17.1", "@types/parse-torrent": "^5.8.4", "@types/passport": "^1.0.12", "@types/passport-jwt": "^3.0.8", @@ -4109,9 +4109,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "12.20.55", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.55.tgz", - "integrity": "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==", + "version": "18.17.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.1.tgz", + "integrity": "sha512-xlR1jahfizdplZYRU59JlUx9uzF1ARa8jbhM11ccpCJya8kvos5jwdm2ZAgxSCwOl0fq21svP18EVwPBXMQudw==", "dev": true }, "node_modules/@types/parse-json": { diff --git a/package.json b/package.json index e1c221705..d1f9c52f8 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "@types/jsonwebtoken": "^9.0.2", "@types/lodash": "^4.14.195", "@types/morgan": "^1.9.4", - "@types/node": "^12.20.55", + "@types/node": "^18.17.1", "@types/parse-torrent": "^5.8.4", "@types/passport": "^1.0.12", "@types/passport-jwt": "^3.0.8", diff --git a/server/routes/api/torrents.ts b/server/routes/api/torrents.ts index 7898c2a27..7cef2078f 100644 --- a/server/routes/api/torrents.ts +++ b/server/routes/api/torrents.ts @@ -7,6 +7,7 @@ import path from 'path'; import rateLimit from 'express-rate-limit'; import sanitize from 'sanitize-filename'; import tar, {Pack} from 'tar-fs'; +import * as fsp from 'node:fs/promises'; import type { AddTorrentByFileOptions, @@ -35,9 +36,17 @@ import { reannounceTorrentsSchema, setTorrentsTagsSchema, } from '../../../shared/schema/api/torrents'; -import {accessDeniedError, fileNotFoundError, isAllowedPath, sanitizePath} from '../../util/fileUtil'; +import { + accessDeniedError, + existAsync, + fileNotFoundError, + isAllowedPath, + isAllowedPathAsync, + sanitizePath, +} from '../../util/fileUtil'; import {getTempPath} from '../../models/TemporaryStorage'; import {getToken} from '../../util/authUtil'; +import {asyncFilter} from '../../util/async'; const getDestination = async ( services: Express.Request['services'], @@ -891,13 +900,13 @@ router.get<{hash: string}>( sanitizePath(path.join(torrentDirectory, content.path)), ); - torrentContentPaths = torrentContentPaths.filter((contentPath) => isAllowedPath(contentPath)); + torrentContentPaths = await asyncFilter(torrentContentPaths, (contentPath) => isAllowedPathAsync(contentPath)); if (torrentContentPaths.length < 1) { const {code, message} = accessDeniedError(); return res.status(403).json({code, message}); } - torrentContentPaths = torrentContentPaths.filter((contentPath) => fs.existsSync(contentPath)); + torrentContentPaths = await asyncFilter(torrentContentPaths, (contentPath) => existAsync(contentPath)); if (torrentContentPaths.length < 1) { const {code, message} = fileNotFoundError(); return res.status(404).json({code, message}); diff --git a/server/util/async.ts b/server/util/async.ts new file mode 100644 index 000000000..35d3f256f --- /dev/null +++ b/server/util/async.ts @@ -0,0 +1,8 @@ +export async function asyncFilter( + arr: Array, + predicate: (item: T, index: number) => Promise, +): Promise> { + const results = await Promise.all(arr.map(predicate)); + + return arr.filter((_v, index) => results[index]); +} diff --git a/server/util/fileUtil.ts b/server/util/fileUtil.ts index c2b6a58c9..42ceace1e 100644 --- a/server/util/fileUtil.ts +++ b/server/util/fileUtil.ts @@ -1,6 +1,7 @@ import fs from 'fs'; import {homedir} from 'os'; import path from 'path'; +import * as fsp from 'node:fs/promises'; import config from '../../config'; @@ -43,6 +44,46 @@ export const isAllowedPath = (resolvedPath: string) => { }); }; +export async function isAllowedPathAsync(resolvedPath: string) { + if (config.allowedPaths == null) { + return true; + } + + let realPath: string | null = null; + let parentPath: string = resolvedPath; + while (realPath == null) { + try { + realPath = await fsp.realpath(parentPath); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') { + parentPath = path.resolve(parentPath, '..'); + } else { + return false; + } + } + } + + return config.allowedPaths.some((allowedPath) => { + if (realPath?.startsWith(allowedPath)) { + return true; + } + return false; + }); +} + +export async function existAsync(path: string): Promise { + try { + await fsp.stat(path); + } catch (err: unknown) { + if ((err as {code?: string}).code === 'ENOENT') { + return false; + } + throw err; + } + + return true; +} + export const sanitizePath = (input?: string): string => { if (typeof input !== 'string') { throw accessDeniedError(); From 88904a25189c0cb42f26611af15e91ea5c7f9cc1 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 27 Jul 2023 07:26:40 +0800 Subject: [PATCH 2/7] fix lint --- server/routes/api/torrents.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/routes/api/torrents.ts b/server/routes/api/torrents.ts index 7cef2078f..ab3de29d1 100644 --- a/server/routes/api/torrents.ts +++ b/server/routes/api/torrents.ts @@ -7,7 +7,6 @@ import path from 'path'; import rateLimit from 'express-rate-limit'; import sanitize from 'sanitize-filename'; import tar, {Pack} from 'tar-fs'; -import * as fsp from 'node:fs/promises'; import type { AddTorrentByFileOptions, From 7bea6b0e0da261e5efc7c3df907e4281762b06d8 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 27 Jul 2023 07:39:16 +0800 Subject: [PATCH 3/7] fix type --- a.js | 1 + server/services/qBittorrent/clientRequestManager.ts | 2 +- server/services/rTorrent/util/scgiUtil.ts | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 a.js diff --git a/a.js b/a.js new file mode 100644 index 000000000..8e89a2e30 --- /dev/null +++ b/a.js @@ -0,0 +1 @@ +console.log(new URLSearchParams({a: undefined, b: 's'}).toString()); diff --git a/server/services/qBittorrent/clientRequestManager.ts b/server/services/qBittorrent/clientRequestManager.ts index 9e8d6100d..53f3e76d7 100644 --- a/server/services/qBittorrent/clientRequestManager.ts +++ b/server/services/qBittorrent/clientRequestManager.ts @@ -483,7 +483,7 @@ class ClientRequestManager { new URLSearchParams({ hashes: hashes.join('|').toLowerCase(), tags: tags?.join(','), - }), + } as Record), { headers: await this.getRequestHeaders(), }, diff --git a/server/services/rTorrent/util/scgiUtil.ts b/server/services/rTorrent/util/scgiUtil.ts index ea075fc58..af93676a0 100644 --- a/server/services/rTorrent/util/scgiUtil.ts +++ b/server/services/rTorrent/util/scgiUtil.ts @@ -27,7 +27,7 @@ export const methodCallXML = (options: net.NetConnectOpts, methodName: string, p const xmlLength = Buffer.byteLength(xml, 'utf8'); stream.on('error', reject); - stream.setEncoding('UTF8'); + stream.setEncoding('utf-8'); const headerItems = [ `CONTENT_LENGTH${NULL_CHAR}${xmlLength}${NULL_CHAR}`, @@ -68,7 +68,7 @@ export const methodCallJSON = (options: net.NetConnectOpts, methodName: string, const jsonLength = Buffer.byteLength(json, 'utf8'); stream.on('error', reject); - stream.setEncoding('UTF8'); + stream.setEncoding('utf-8'); const headerItems = [ `CONTENT_LENGTH${NULL_CHAR}${jsonLength}${NULL_CHAR}`, From c6c5349bfc56cbcb72382c248e4bb6b6ae7ac447 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 27 Jul 2023 07:40:28 +0800 Subject: [PATCH 4/7] unexpected file --- a.js | 1 - 1 file changed, 1 deletion(-) delete mode 100644 a.js diff --git a/a.js b/a.js deleted file mode 100644 index 8e89a2e30..000000000 --- a/a.js +++ /dev/null @@ -1 +0,0 @@ -console.log(new URLSearchParams({a: undefined, b: 's'}).toString()); From be0c6cd191dc9215e8533c126439f7faa0eb53f8 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 27 Jul 2023 07:48:11 +0800 Subject: [PATCH 5/7] better type --- server/util/fileUtil.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/util/fileUtil.ts b/server/util/fileUtil.ts index 42ceace1e..018346e93 100644 --- a/server/util/fileUtil.ts +++ b/server/util/fileUtil.ts @@ -75,7 +75,7 @@ export async function existAsync(path: string): Promise { try { await fsp.stat(path); } catch (err: unknown) { - if ((err as {code?: string}).code === 'ENOENT') { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { return false; } throw err; From 8a3aa85ac91aa96ddf313d3dbd508b179b9efda5 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Sun, 30 Jul 2023 03:37:54 +0800 Subject: [PATCH 6/7] revert nodejs type --- package-lock.json | 8 ++++---- package.json | 2 +- server/services/qBittorrent/clientRequestManager.ts | 2 +- server/services/rTorrent/util/scgiUtil.ts | 4 ++-- server/util/fileUtil.ts | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 33e3bfa7f..00c5ce9cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,7 +52,7 @@ "@types/jsonwebtoken": "^9.0.2", "@types/lodash": "^4.14.195", "@types/morgan": "^1.9.4", - "@types/node": "^18.17.1", + "@types/node": "^12.20.55", "@types/parse-torrent": "^5.8.4", "@types/passport": "^1.0.12", "@types/passport-jwt": "^3.0.8", @@ -4534,9 +4534,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.17.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.1.tgz", - "integrity": "sha512-xlR1jahfizdplZYRU59JlUx9uzF1ARa8jbhM11ccpCJya8kvos5jwdm2ZAgxSCwOl0fq21svP18EVwPBXMQudw==", + "version": "12.20.55", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.55.tgz", + "integrity": "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==", "dev": true }, "node_modules/@types/parse-json": { diff --git a/package.json b/package.json index 1c3ecf049..5a8189570 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "@types/jsonwebtoken": "^9.0.2", "@types/lodash": "^4.14.195", "@types/morgan": "^1.9.4", - "@types/node": "^18.17.1", + "@types/node": "^12.20.55", "@types/parse-torrent": "^5.8.4", "@types/passport": "^1.0.12", "@types/passport-jwt": "^3.0.8", diff --git a/server/services/qBittorrent/clientRequestManager.ts b/server/services/qBittorrent/clientRequestManager.ts index 53f3e76d7..9e8d6100d 100644 --- a/server/services/qBittorrent/clientRequestManager.ts +++ b/server/services/qBittorrent/clientRequestManager.ts @@ -483,7 +483,7 @@ class ClientRequestManager { new URLSearchParams({ hashes: hashes.join('|').toLowerCase(), tags: tags?.join(','), - } as Record), + }), { headers: await this.getRequestHeaders(), }, diff --git a/server/services/rTorrent/util/scgiUtil.ts b/server/services/rTorrent/util/scgiUtil.ts index af93676a0..ea075fc58 100644 --- a/server/services/rTorrent/util/scgiUtil.ts +++ b/server/services/rTorrent/util/scgiUtil.ts @@ -27,7 +27,7 @@ export const methodCallXML = (options: net.NetConnectOpts, methodName: string, p const xmlLength = Buffer.byteLength(xml, 'utf8'); stream.on('error', reject); - stream.setEncoding('utf-8'); + stream.setEncoding('UTF8'); const headerItems = [ `CONTENT_LENGTH${NULL_CHAR}${xmlLength}${NULL_CHAR}`, @@ -68,7 +68,7 @@ export const methodCallJSON = (options: net.NetConnectOpts, methodName: string, const jsonLength = Buffer.byteLength(json, 'utf8'); stream.on('error', reject); - stream.setEncoding('utf-8'); + stream.setEncoding('UTF8'); const headerItems = [ `CONTENT_LENGTH${NULL_CHAR}${jsonLength}${NULL_CHAR}`, diff --git a/server/util/fileUtil.ts b/server/util/fileUtil.ts index 018346e93..0e7617e15 100644 --- a/server/util/fileUtil.ts +++ b/server/util/fileUtil.ts @@ -1,7 +1,7 @@ import fs from 'fs'; +import {promises as fsp} from 'fs'; import {homedir} from 'os'; import path from 'path'; -import * as fsp from 'node:fs/promises'; import config from '../../config'; From ab6faa38f84342dab74a0e245d4c3327e940c021 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Sun, 30 Jul 2023 03:49:17 +0800 Subject: [PATCH 7/7] use for loop await --- server/util/async.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/util/async.ts b/server/util/async.ts index 35d3f256f..b104bf99d 100644 --- a/server/util/async.ts +++ b/server/util/async.ts @@ -1,8 +1,14 @@ export async function asyncFilter( - arr: Array, + array: Array, predicate: (item: T, index: number) => Promise, ): Promise> { - const results = await Promise.all(arr.map(predicate)); + const results: T[] = []; - return arr.filter((_v, index) => results[index]); + for (const [index, item] of array.entries()) { + if (await predicate(item, index)) { + results.push(item); + } + } + + return results; }