From b25f2ad51881198982c886d179d23669a9b4074a Mon Sep 17 00:00:00 2001 From: Alexander Alemayhu Date: Sat, 31 Aug 2024 17:15:22 +0200 Subject: [PATCH] Revert "Revert "chore: remove debug option" (#1590)" This reverts commit 322120f868fa439e0e37e2c250e4224e34d3ed22. --- src/lib/parser/DeckParser.ts | 40 +++++-- src/lib/parser/PrepareDeck.ts | 12 +-- src/test/test-utils.ts | 7 +- .../uploads/GeneratePackagesUseCase.ts | 101 ++---------------- src/usecases/uploads/getPackagesFromZip.ts | 68 ++++++++++++ src/usecases/uploads/worker.ts | 56 ++++++++++ 6 files changed, 176 insertions(+), 108 deletions(-) create mode 100644 src/usecases/uploads/getPackagesFromZip.ts create mode 100644 src/usecases/uploads/worker.ts diff --git a/src/lib/parser/DeckParser.ts b/src/lib/parser/DeckParser.ts index cc7fe66fc..6e96ca4b2 100644 --- a/src/lib/parser/DeckParser.ts +++ b/src/lib/parser/DeckParser.ts @@ -23,6 +23,14 @@ import { isFileNameEqual } from '../storage/types'; import { isImageFileEmbedable, isMarkdownFile } from '../storage/checks'; import { getFileContents } from './getFileContents'; import { handleNestedBulletPointsInMarkdown } from './handleNestedBulletPointsInMarkdown'; +import { checkFlashcardsLimits } from '../User/checkFlashcardsLimits'; + +export interface DeckParserInput { + name: string; + settings: Settings; + files: File[]; + noLimits: boolean; +} export class DeckParser { globalTags: cheerio.Cheerio | null; @@ -35,22 +43,27 @@ export class DeckParser { files: File[]; + noLimits: boolean; + public get name() { return this.payload[0].name; } - constructor(name: string, settings: Settings, files: File[]) { - this.settings = settings; - this.files = files || []; - this.firstDeckName = name; + constructor(input: DeckParserInput) { + this.settings = input.settings; + this.files = input.files || []; + this.firstDeckName = input.name; + this.noLimits = input.noLimits; this.globalTags = null; - const firstFile = this.files.find((file) => isFileNameEqual(file, name)); + const firstFile = this.files.find((file) => + isFileNameEqual(file, input.name) + ); - if (this.settings.nestedBulletPoints && isMarkdownFile(name)) { + if (this.settings.nestedBulletPoints && isMarkdownFile(input.name)) { const contents = getFileContents(firstFile, false); this.payload = handleNestedBulletPointsInMarkdown( - name, + input.name, contents?.toString(), this.settings.deckName, [], @@ -60,7 +73,7 @@ export class DeckParser { const contents = getFileContents(firstFile, true); this.payload = contents ? this.handleHTML( - name, + input.name, contents.toString(), this.settings.deckName || '', [] @@ -571,6 +584,8 @@ export class DeckParser { const parentUL = p; const parentClass = p.attr('class') || ''; + this.checkLimits(cards.length, []); + if (this.settings.toggleMode === 'open_toggle') { dom('details').attr('open', ''); } else if (this.settings.toggleMode === 'close_toggle') { @@ -651,10 +666,19 @@ export class DeckParser { lists.forEach((list) => { for (const child of dom(list).find('li')) { + this.checkLimits(cards.length, []); cards.push(new Note(dom(child).html() ?? '', '')); } }); return cards; } + + private checkLimits(cards: number, decks: Deck[]) { + checkFlashcardsLimits({ + cards: cards, + decks: decks, + paying: this.noLimits, + }); + } } diff --git a/src/lib/parser/PrepareDeck.ts b/src/lib/parser/PrepareDeck.ts index e91dcf7dd..72fea9fd2 100644 --- a/src/lib/parser/PrepareDeck.ts +++ b/src/lib/parser/PrepareDeck.ts @@ -1,7 +1,5 @@ -import { File } from '../anki/zip'; -import Settings from './Settings'; import getDeckFilename from '../anki/getDeckFilename'; -import { DeckParser } from './DeckParser'; +import { DeckParser, DeckParserInput } from './DeckParser'; import Deck from './Deck'; interface PrepareDeckResult { @@ -11,16 +9,14 @@ interface PrepareDeckResult { } export async function PrepareDeck( - fileName: string, - files: File[], - settings: Settings + input: DeckParserInput ): Promise { - const parser = new DeckParser(fileName, settings, files); + const parser = new DeckParser(input); if (parser.totalCardCount() === 0) { const apkg = await parser.tryExperimental(); return { - name: getDeckFilename(parser.name ?? fileName), + name: getDeckFilename(parser.name ?? input.name), apkg, deck: parser.payload, }; diff --git a/src/test/test-utils.ts b/src/test/test-utils.ts index adbcdd077..8c17e51aa 100644 --- a/src/test/test-utils.ts +++ b/src/test/test-utils.ts @@ -19,7 +19,12 @@ function loadFixture(fileName: string) { function configureParser(fileName: string, opts: Settings) { const info = loadFixture(fileName); - return new DeckParser(fileName, opts, info); + return new DeckParser({ + name: fileName, + settings: opts, + files: info, + noLimits: true, + }); } export async function getDeck(fileName: string, opts: Settings) { diff --git a/src/usecases/uploads/GeneratePackagesUseCase.ts b/src/usecases/uploads/GeneratePackagesUseCase.ts index f3808e014..6890ead8b 100644 --- a/src/usecases/uploads/GeneratePackagesUseCase.ts +++ b/src/usecases/uploads/GeneratePackagesUseCase.ts @@ -1,108 +1,27 @@ -import fs from 'fs'; - -import { ZipHandler } from '../../lib/anki/zip'; import Package from '../../lib/parser/Package'; import Settings from '../../lib/parser/Settings'; -import { - isCSVFile, - isHTMLFile, - isMarkdownFile, - isPlainText, - isZIPFile, -} from '../../lib/storage/checks'; import { UploadedFile } from '../../lib/storage/types'; - -import { Body } from 'aws-sdk/clients/s3'; -import { PrepareDeck } from '../../lib/parser/PrepareDeck'; -import { checkFlashcardsLimits } from '../../lib/User/checkFlashcardsLimits'; +import { Worker } from 'worker_threads'; +import path from 'path'; export interface PackageResult { packages: Package[]; } -export const isFileSupported = (filename: string) => - isHTMLFile(filename) ?? - isMarkdownFile(filename) ?? - isPlainText(filename) ?? - isCSVFile(filename); - -const getPackagesFromZip = async ( - fileContents: Body | undefined, - paying: boolean, - settings: Settings -): Promise => { - const zipHandler = new ZipHandler(); - const packages = []; - - if (!fileContents) { - return { packages: [] }; - } - - zipHandler.build(fileContents as Uint8Array, paying); - - const fileNames = zipHandler.getFileNames(); - - let cardCount = 0; - for (const fileName of fileNames) { - if (isFileSupported(fileName)) { - const deck = await PrepareDeck(fileName, zipHandler.files, settings); - - if (deck) { - packages.push(new Package(deck.name, deck.apkg)); - cardCount += deck.deck.reduce((acc, d) => acc + d.cards.length, 0); - - // Checking the limit in place while iterating through the decks - checkFlashcardsLimits({ - cards: 0, - decks: deck.deck, - paying, - }); - } - } - - // Checking the limit in place while iterating through the files - checkFlashcardsLimits({ - cards: cardCount, - paying: paying, - }); - } - - return { packages }; -}; - class GeneratePackagesUseCase { - async execute( + execute( paying: boolean, files: UploadedFile[], settings: Settings ): Promise { - let packages: Package[] = []; + return new Promise((resolve, reject) => { + const data = { paying, files, settings }; + const workerPath = path.resolve(__dirname, './worker.js'); + const worker = new Worker(workerPath, { workerData: { data } }); - for (const file of files) { - const fileContents = file.path ? fs.readFileSync(file.path) : file.buffer; - const filename = file.originalname; - const key = file.key; - - if (isFileSupported(filename)) { - const d = await PrepareDeck( - filename, - [{ name: filename, contents: fileContents }], - settings - ); - if (d) { - const pkg = new Package(d.name, d.apkg); - packages = packages.concat(pkg); - } - } else if (isZIPFile(filename) || isZIPFile(key)) { - const { packages: extraPackages } = await getPackagesFromZip( - fileContents, - paying, - settings - ); - packages = packages.concat(extraPackages); - } - } - return { packages }; + worker.on('message', (result: PackageResult) => resolve(result)); + worker.on('error', (error) => reject(error)); + }); } } diff --git a/src/usecases/uploads/getPackagesFromZip.ts b/src/usecases/uploads/getPackagesFromZip.ts new file mode 100644 index 000000000..dd25b7093 --- /dev/null +++ b/src/usecases/uploads/getPackagesFromZip.ts @@ -0,0 +1,68 @@ +import { Body } from 'aws-sdk/clients/s3'; +import Settings from '../../lib/parser/Settings'; +import { ZipHandler } from '../../lib/anki/zip'; +import { PrepareDeck } from '../../lib/parser/PrepareDeck'; +import Package from '../../lib/parser/Package'; +import { checkFlashcardsLimits } from '../../lib/User/checkFlashcardsLimits'; +import { PackageResult } from './GeneratePackagesUseCase'; +import { + isCSVFile, + isHTMLFile, + isMarkdownFile, + isPlainText, +} from '../../lib/storage/checks'; + +export const isFileSupported = (filename: string) => + isHTMLFile(filename) ?? + isMarkdownFile(filename) ?? + isPlainText(filename) ?? + isCSVFile(filename); + +export const getPackagesFromZip = async ( + fileContents: Body | undefined, + paying: boolean, + settings: Settings +): Promise => { + const zipHandler = new ZipHandler(); + const packages = []; + + if (!fileContents) { + return { packages: [] }; + } + + zipHandler.build(fileContents as Uint8Array, paying); + + const fileNames = zipHandler.getFileNames(); + + let cardCount = 0; + for (const fileName of fileNames) { + if (isFileSupported(fileName)) { + const deck = await PrepareDeck({ + name: fileName, + files: zipHandler.files, + settings, + noLimits: paying, + }); + + if (deck) { + packages.push(new Package(deck.name, deck.apkg)); + cardCount += deck.deck.reduce((acc, d) => acc + d.cards.length, 0); + + // Checking the limit in place while iterating through the decks + checkFlashcardsLimits({ + cards: 0, + decks: deck.deck, + paying, + }); + } + } + + // Checking the limit in place while iterating through the files + checkFlashcardsLimits({ + cards: cardCount, + paying: paying, + }); + } + + return { packages }; +}; diff --git a/src/usecases/uploads/worker.ts b/src/usecases/uploads/worker.ts new file mode 100644 index 000000000..290f3310a --- /dev/null +++ b/src/usecases/uploads/worker.ts @@ -0,0 +1,56 @@ +import { parentPort, workerData } from 'worker_threads'; +import { UploadedFile } from '../../lib/storage/types'; +import Settings from '../../lib/parser/Settings'; +import Package from '../../lib/parser/Package'; +import fs from 'fs'; +import { PrepareDeck } from '../../lib/parser/PrepareDeck'; +import { isZIPFile } from '../../lib/storage/checks'; +import { getPackagesFromZip, isFileSupported } from './getPackagesFromZip'; + +interface GenerationData { + paying: boolean; + files: UploadedFile[]; + settings: Settings; +} + +function doGenerationWork(data: GenerationData) { + console.log('doGenerationWork'); + return new Promise(async (resolve) => { + console.log('starting generation'); + const { paying, files, settings } = data; + let packages: Package[] = []; + + for (const file of files) { + const fileContents = file.path ? fs.readFileSync(file.path) : file.buffer; + const filename = file.originalname; + const key = file.key; + + if (isFileSupported(filename)) { + const d = await PrepareDeck({ + name: filename, + files: [{ name: filename, contents: fileContents }], + settings, + noLimits: paying, + }); + if (d) { + const pkg = new Package(d.name, d.apkg); + packages = packages.concat(pkg); + } + } else if (isZIPFile(filename) || isZIPFile(key)) { + const { packages: extraPackages } = await getPackagesFromZip( + fileContents, + paying, + settings + ); + packages = packages.concat(extraPackages); + } + } + resolve({ packages }); + }); +} + +doGenerationWork(workerData.data) + .then((result) => { + parentPort?.postMessage(result); + }) + .catch(parentPort?.postMessage);