diff --git a/package-lock.json b/package-lock.json
index 6a176d639..c291d3b7c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -26,7 +26,6 @@
"get-notion-object-title": "^0.2.0",
"html-to-text": "^9.0.4",
"jsonwebtoken": "^9.0.0",
- "jszip": "^3.10.1",
"knex": "^3.1.0",
"metascraper": "^5.34.7",
"metascraper-description": "^5.39.0",
@@ -7251,11 +7250,6 @@
"node": ">=0.10.0"
}
},
- "node_modules/immediate": {
- "version": "3.0.6",
- "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
- "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps="
- },
"node_modules/import-fresh": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
@@ -8641,17 +8635,6 @@
"npm": ">=6"
}
},
- "node_modules/jszip": {
- "version": "3.10.1",
- "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
- "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
- "dependencies": {
- "lie": "~3.3.0",
- "pako": "~1.0.2",
- "readable-stream": "~2.3.6",
- "setimmediate": "^1.0.5"
- }
- },
"node_modules/jwa": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
@@ -8797,14 +8780,6 @@
"node": ">= 0.8.0"
}
},
- "node_modules/lie": {
- "version": "3.3.0",
- "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
- "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
- "dependencies": {
- "immediate": "~3.0.5"
- }
- },
"node_modules/lilconfig": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.0.0.tgz",
@@ -10266,11 +10241,6 @@
"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
"integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ=="
},
- "node_modules/pako": {
- "version": "1.0.11",
- "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
- "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
- },
"node_modules/parent-module": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@@ -11285,11 +11255,6 @@
"node": ">= 0.4"
}
},
- "node_modules/setimmediate": {
- "version": "1.0.5",
- "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
- "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="
- },
"node_modules/setprototypeof": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
diff --git a/package.json b/package.json
index 7cd4634ff..2326a6765 100644
--- a/package.json
+++ b/package.json
@@ -47,7 +47,6 @@
"get-notion-object-title": "^0.2.0",
"html-to-text": "^9.0.4",
"jsonwebtoken": "^9.0.0",
- "jszip": "^3.10.1",
"knex": "^3.1.0",
"metascraper": "^5.34.7",
"metascraper-description": "^5.39.0",
diff --git a/src/controllers/DownloadController.ts b/src/controllers/DownloadController.ts
index 38f3dbda1..872451f19 100644
--- a/src/controllers/DownloadController.ts
+++ b/src/controllers/DownloadController.ts
@@ -61,7 +61,10 @@ class DownloadController {
return;
}
- const page = DownloadPage({ id, files });
+ const page = DownloadPage({
+ id,
+ files: files.filter((file) => file.endsWith('.apkg')),
+ });
res.send(page);
});
} else {
diff --git a/src/controllers/SimpleUploadController/SimpleUploadController.ts b/src/controllers/SimpleUploadController/SimpleUploadController.ts
deleted file mode 100644
index 97ce5edc2..000000000
--- a/src/controllers/SimpleUploadController/SimpleUploadController.ts
+++ /dev/null
@@ -1,63 +0,0 @@
-import express from 'express';
-
-import { sendError } from '../../lib/error/sendError';
-import { getLimitMessage } from '../../lib/misc/getLimitMessage';
-import { UploadedFile } from '../../lib/storage/types';
-
-import { getUploadHandler } from '../../lib/misc/GetUploadHandler';
-import { createPackages } from './createPackages';
-import { CreatedDeck, createResponse } from './createResponse';
-import { isPaying } from '../../lib/isPaying';
-
-const getPayingErrorMessage = () => {
- return "There was an unknown error with your upload. Please try again. If the problem persists, please contact support@2anki.net.";
-};
-
-class SimpleUploadController {
- async handleUpload(req: express.Request, res: express.Response) {
- try {
- const packages = await createPackages(
- req.files as UploadedFile[],
- isPaying(res.locals),
- req.body
- );
- const response: CreatedDeck[] = createResponse(packages);
- return res.json(response);
- } catch (err) {
- if (err instanceof Error) {
- return res.json({
- error: err.message,
- });
- }
- }
- }
-
- file(req: express.Request, res: express.Response) {
- try {
- console.info('uploading file');
- const handleUploadEndpoint = getUploadHandler(res);
-
- handleUploadEndpoint(req, res, async (error) => {
- if (error) {
- let msg = error.message;
- if (msg === 'File too large' && !isPaying(res.locals)) {
- msg = getLimitMessage();
- } else if (isPaying(res.locals)) {
- msg = getPayingErrorMessage();
- console.info('paying customer issue');
- sendError(error);
- } else {
- sendError(error);
- }
- return res.status(500).send(msg);
- }
- await this.handleUpload(req, res);
- });
- } catch (error) {
- sendError(error);
- res.status(400);
- }
- }
-}
-
-export default SimpleUploadController;
diff --git a/src/controllers/SimpleUploadController/createPackages.ts b/src/controllers/SimpleUploadController/createPackages.ts
deleted file mode 100644
index deaec3b14..000000000
--- a/src/controllers/SimpleUploadController/createPackages.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-import Settings from '../../lib/parser/Settings';
-import GeneratePackagesUseCase from '../../usecases/uploads/GeneratePackagesUseCase';
-import { UploadedFile } from '../../lib/storage/types';
-
-export const createPackages = async (
- files: UploadedFile[],
- paying: boolean,
- body: { [key: string]: string } = {}
-) => {
- const settings = new Settings(body);
-
- const useCase = new GeneratePackagesUseCase();
- const { packages } = await useCase.execute(paying, files, settings);
-
- return packages;
-};
diff --git a/src/controllers/SimpleUploadController/createResponse.ts b/src/controllers/SimpleUploadController/createResponse.ts
deleted file mode 100644
index 10e65d5d2..000000000
--- a/src/controllers/SimpleUploadController/createResponse.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-import path from 'path';
-import fs from 'fs';
-
-import Package from '../../lib/parser/Package';
-import Workspace from '../../lib/parser/WorkSpace';
-
-export interface CreatedDeck {
- name: string;
- link: string;
-}
-export const createResponse = (packages: Package[]) => {
- const workspace = new Workspace(true, 'fs');
- const basePath = `/download/${workspace.id}`;
- const createdDecks = [];
- for (const pkg of packages) {
- const p = path.join(workspace.location, pkg.name);
- fs.writeFileSync(p, pkg.apkg);
- createdDecks.push({
- name: pkg.name,
- link: `${basePath}/${pkg.name}`,
- });
- }
- return createdDecks;
-};
diff --git a/src/lib/anki/getDeckFilename.test.ts b/src/lib/anki/getDeckFilename.test.ts
index 3d39850bc..cdf31fa49 100644
--- a/src/lib/anki/getDeckFilename.test.ts
+++ b/src/lib/anki/getDeckFilename.test.ts
@@ -10,7 +10,5 @@ test("does not append .apkg extension if it's already there", () => {
});
test("uses package name if it's available", () => {
- expect(getDeckFilename(new Package('foo', Buffer.alloc(0)))).toEqual(
- 'foo.apkg'
- );
+ expect(getDeckFilename(new Package('foo'))).toEqual('foo.apkg');
});
diff --git a/src/lib/anki/zip.tsx b/src/lib/anki/zip.tsx
index 29d78cedf..f2223f93c 100644
--- a/src/lib/anki/zip.tsx
+++ b/src/lib/anki/zip.tsx
@@ -1,11 +1,8 @@
-import JSZip from 'jszip';
import { strFromU8, unzipSync } from 'fflate';
-import Package from '../parser/Package';
import { Body } from 'aws-sdk/clients/s3';
import { renderToStaticMarkup } from 'react-dom/server';
import { getUploadLimits } from '../misc/getUploadLimits';
import { isHTMLFile, isMarkdownFile } from '../storage/checks';
-import getDeckFilename from './getDeckFilename';
interface File {
name: string;
@@ -60,17 +57,6 @@ class ZipHandler {
getFileNames() {
return this.fileNames;
}
-
- static toZip(decks: Package[], advertisment: string | null) {
- const zip = new JSZip();
- for (const d of decks) {
- zip.file(getDeckFilename(d), d.apkg);
- }
- if (advertisment) {
- zip.file('README.html', advertisment);
- }
- return zip.generateAsync({ type: 'nodebuffer' });
- }
}
export { ZipHandler, File };
diff --git a/src/lib/parser/DeckParser.ts b/src/lib/parser/DeckParser.ts
index cc7fe66fc..b9425cc3c 100644
--- a/src/lib/parser/DeckParser.ts
+++ b/src/lib/parser/DeckParser.ts
@@ -23,6 +23,15 @@ import { isFileNameEqual } from '../storage/types';
import { isImageFileEmbedable, isMarkdownFile } from '../storage/checks';
import { getFileContents } from './getFileContents';
import { handleNestedBulletPointsInMarkdown } from './handleNestedBulletPointsInMarkdown';
+import { checkFlashcardsLimits } from '../User/checkFlashcardsLimits';
+
+export interface DeckParserInput {
+ name: string;
+ settings: Settings;
+ files: File[];
+ noLimits: boolean;
+ workspace: Workspace;
+}
export class DeckParser {
globalTags: cheerio.Cheerio | null;
@@ -35,22 +44,27 @@ export class DeckParser {
files: File[];
+ noLimits: boolean;
+
public get name() {
return this.payload[0].name;
}
- constructor(name: string, settings: Settings, files: File[]) {
- this.settings = settings;
- this.files = files || [];
- this.firstDeckName = name;
+ constructor(input: DeckParserInput) {
+ this.settings = input.settings;
+ this.files = input.files || [];
+ this.firstDeckName = input.name;
+ this.noLimits = input.noLimits;
this.globalTags = null;
- const firstFile = this.files.find((file) => isFileNameEqual(file, name));
+ const firstFile = this.files.find((file) =>
+ isFileNameEqual(file, input.name)
+ );
- if (this.settings.nestedBulletPoints && isMarkdownFile(name)) {
+ if (this.settings.nestedBulletPoints && isMarkdownFile(input.name)) {
const contents = getFileContents(firstFile, false);
this.payload = handleNestedBulletPointsInMarkdown(
- name,
+ input.name,
contents?.toString(),
this.settings.deckName,
[],
@@ -60,7 +74,7 @@ export class DeckParser {
const contents = getFileContents(firstFile, true);
this.payload = contents
? this.handleHTML(
- name,
+ input.name,
contents.toString(),
this.settings.deckName || '',
[]
@@ -341,8 +355,7 @@ export class DeckParser {
return card;
}
- build() {
- const ws = new Workspace(true, 'fs');
+ build(ws: Workspace) {
const exporter = this.setupExporter(this.payload, ws.location);
for (const d of this.payload) {
@@ -462,9 +475,8 @@ export class DeckParser {
return exporter.save();
}
- tryExperimental() {
+ tryExperimental(ws: Workspace) {
const fallback = new FallbackParser(this.files);
- const ws = new Workspace(true, 'fs');
const exporter = this.setupExporter(this.payload, ws.location);
this.payload = fallback.run(this.settings);
@@ -571,6 +583,8 @@ export class DeckParser {
const parentUL = p;
const parentClass = p.attr('class') || '';
+ this.checkLimits(cards.length, []);
+
if (this.settings.toggleMode === 'open_toggle') {
dom('details').attr('open', '');
} else if (this.settings.toggleMode === 'close_toggle') {
@@ -651,10 +665,19 @@ export class DeckParser {
lists.forEach((list) => {
for (const child of dom(list).find('li')) {
+ this.checkLimits(cards.length, []);
cards.push(new Note(dom(child).html() ?? '', ''));
}
});
return cards;
}
+
+ private checkLimits(cards: number, decks: Deck[]) {
+ checkFlashcardsLimits({
+ cards: cards,
+ decks: decks,
+ paying: this.noLimits,
+ });
+ }
}
diff --git a/src/lib/parser/Package.ts b/src/lib/parser/Package.ts
index 34b36b68e..4ed75b8c9 100644
--- a/src/lib/parser/Package.ts
+++ b/src/lib/parser/Package.ts
@@ -1,11 +1,8 @@
class Package {
name: string;
- apkg: Buffer;
-
- constructor(name: string, apkg: Buffer) {
+ constructor(name: string) {
this.name = name;
- this.apkg = apkg;
}
}
diff --git a/src/lib/parser/PrepareDeck.ts b/src/lib/parser/PrepareDeck.ts
index e91dcf7dd..65d57de2a 100644
--- a/src/lib/parser/PrepareDeck.ts
+++ b/src/lib/parser/PrepareDeck.ts
@@ -1,7 +1,5 @@
-import { File } from '../anki/zip';
-import Settings from './Settings';
import getDeckFilename from '../anki/getDeckFilename';
-import { DeckParser } from './DeckParser';
+import { DeckParser, DeckParserInput } from './DeckParser';
import Deck from './Deck';
interface PrepareDeckResult {
@@ -11,22 +9,20 @@ interface PrepareDeckResult {
}
export async function PrepareDeck(
- fileName: string,
- files: File[],
- settings: Settings
+ input: DeckParserInput
): Promise {
- const parser = new DeckParser(fileName, settings, files);
+ const parser = new DeckParser(input);
if (parser.totalCardCount() === 0) {
- const apkg = await parser.tryExperimental();
+ const apkg = await parser.tryExperimental(input.workspace);
return {
- name: getDeckFilename(parser.name ?? fileName),
+ name: getDeckFilename(parser.name ?? input.name),
apkg,
deck: parser.payload,
};
}
- const apkg = await parser.build();
+ const apkg = await parser.build(input.workspace);
return {
name: getDeckFilename(parser.name),
apkg,
diff --git a/src/lib/parser/WorkSpace.ts b/src/lib/parser/WorkSpace.ts
index 7c5288544..768b3deae 100644
--- a/src/lib/parser/WorkSpace.ts
+++ b/src/lib/parser/WorkSpace.ts
@@ -22,6 +22,19 @@ class Workspace {
fs.mkdirSync(this.location, { recursive: true });
}
}
+
+ public getFirstAPKG(): Promise {
+ return new Promise((resolve, reject) => {
+ fs.readdir(this.location, (err, files) => {
+ const apkg = files.find((file) => file.endsWith('.apkg'));
+ if (apkg) {
+ resolve(fs.readFileSync(path.join(this.location, apkg)));
+ } else {
+ reject(null);
+ }
+ });
+ });
+ }
}
export default Workspace;
diff --git a/src/pages/DownloadPage.tsx b/src/pages/DownloadPage.tsx
index de8c10763..2c18dcd68 100644
--- a/src/pages/DownloadPage.tsx
+++ b/src/pages/DownloadPage.tsx
@@ -44,6 +44,7 @@ export const DownloadPage = ({ id, files }: DownloadPageProps) => {
},
downloadItemLink: {},
};
+ const apkgFiles = files.filter((file) => file.endsWith('.apkg'));
return ReactDOMServer.renderToStaticMarkup(
@@ -56,7 +57,7 @@ export const DownloadPage = ({ id, files }: DownloadPageProps) => {
- {files.map((file) => (
+ {apkgFiles.map((file) => (
-
{file}
{
- const controller = new SimpleUploadController();
- const router = express.Router();
-
- router.post('/api/simple-upload/file', RequireAllowedOrigin, (req, res) =>
- controller.file(req, res)
- );
-
- return router;
-};
-
-export default UploadRouter;
diff --git a/src/server.ts b/src/server.ts
index 96f70e2be..6fd9f0b6f 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -27,7 +27,6 @@ import downloadRouter from './routes/DownloadRouter';
import favoriteRouter from './routes/FavoriteRouter';
import templatesRouter from './routes/TemplatesRouter';
import defaultRouter from './routes/DefaultRouter';
-import simpleUploadRouter from './routes/SimpleUploadRouter';
import webhookRouter from './routes/WebhookRouter';
import { sendError } from './lib/error/sendError';
@@ -75,7 +74,6 @@ const serve = async () => {
app.use(downloadRouter());
app.use(favoriteRouter());
app.use(templatesRouter());
- app.use(simpleUploadRouter());
// Note: this has to be the last router
app.use(defaultRouter());
diff --git a/src/services/UploadService.ts b/src/services/UploadService.ts
index d6fcffcd8..d6fcf8cd3 100644
--- a/src/services/UploadService.ts
+++ b/src/services/UploadService.ts
@@ -1,6 +1,3 @@
-import fs from 'fs';
-import path from 'path';
-
import express from 'express';
import { IUploadRepository } from '../data_layer/UploadRespository';
@@ -12,7 +9,6 @@ import StorageHandler from '../lib/storage/StorageHandler';
import { UploadedFile } from '../lib/storage/types';
import GeneratePackagesUseCase from '../usecases/uploads/GeneratePackagesUseCase';
import { toText } from './NotionService/BlockHandler/helpers/deckNameToText';
-import { getSafeFilename } from '../lib/getSafeFilename';
import { isPaying } from '../lib/isPaying';
import { isLimitError } from '../lib/misc/isLimitError';
import { handleUploadLimitError } from '../controllers/Upload/helpers/handleUploadLimitError';
@@ -35,22 +31,27 @@ class UploadService {
let payload;
let plen;
const settings = new Settings(req.body || {});
+ const ws = new Workspace(true, 'fs');
const useCase = new GeneratePackagesUseCase();
const { packages } = await useCase.execute(
isPaying(res.locals),
req.files as UploadedFile[],
- settings
+ settings,
+ ws
);
+ console.log('packages', packages);
+
const first = packages[0];
if (packages.length === 1) {
- if (!first.apkg) {
+ const apkg = await ws.getFirstAPKG();
+ if (!apkg) {
const name = first ? first.name : 'untitled';
throw new Error(`Could not produce APKG for ${name}`);
}
- payload = first.apkg;
- plen = Buffer.byteLength(first.apkg);
+ payload = apkg;
+ plen = Buffer.byteLength(apkg);
res.set('Content-Type', 'application/apkg');
res.set('Content-Length', plen.toString());
first.name = toText(first.name);
@@ -64,14 +65,7 @@ class UploadService {
res.attachment(`/${first.name}`);
return res.status(200).send(payload);
} else if (packages.length > 1) {
- const workspace = new Workspace(true, 'fs');
-
- for (const pkg of packages) {
- const p = path.join(workspace.location, getSafeFilename(pkg.name));
- fs.writeFileSync(p, pkg.apkg);
- }
-
- const url = `/download/${workspace.id}`;
+ const url = `/download/${ws.id}`;
res.status(300);
return res.redirect(url);
} else {
diff --git a/src/test/test-utils.ts b/src/test/test-utils.ts
index adbcdd077..11f1d3a24 100644
--- a/src/test/test-utils.ts
+++ b/src/test/test-utils.ts
@@ -6,6 +6,7 @@ import fs from 'fs';
import { DeckParser } from '../lib/parser/DeckParser';
import Settings from '../lib/parser/Settings';
+import Workspace from '../lib/parser/WorkSpace';
function mockPayload(name: string, contents: string) {
return [{ name, contents }];
@@ -19,12 +20,18 @@ function loadFixture(fileName: string) {
function configureParser(fileName: string, opts: Settings) {
const info = loadFixture(fileName);
- return new DeckParser(fileName, opts, info);
+ return new DeckParser({
+ name: fileName,
+ settings: opts,
+ files: info,
+ noLimits: true,
+ workspace: new Workspace(true, 'fs'),
+ });
}
export async function getDeck(fileName: string, opts: Settings) {
const p = configureParser(fileName, opts);
- await p.build();
+ await p.build(new Workspace(true, 'fs'));
return p.payload[0];
}
export const pageId = '3ce6b147ac8a425f836b51cc21825b85';
diff --git a/src/usecases/uploads/GeneratePackagesUseCase.ts b/src/usecases/uploads/GeneratePackagesUseCase.ts
index f3808e014..c1c3a4d58 100644
--- a/src/usecases/uploads/GeneratePackagesUseCase.ts
+++ b/src/usecases/uploads/GeneratePackagesUseCase.ts
@@ -1,108 +1,29 @@
-import fs from 'fs';
-
-import { ZipHandler } from '../../lib/anki/zip';
import Package from '../../lib/parser/Package';
import Settings from '../../lib/parser/Settings';
-import {
- isCSVFile,
- isHTMLFile,
- isMarkdownFile,
- isPlainText,
- isZIPFile,
-} from '../../lib/storage/checks';
import { UploadedFile } from '../../lib/storage/types';
-
-import { Body } from 'aws-sdk/clients/s3';
-import { PrepareDeck } from '../../lib/parser/PrepareDeck';
-import { checkFlashcardsLimits } from '../../lib/User/checkFlashcardsLimits';
+import { Worker } from 'worker_threads';
+import path from 'path';
+import Workspace from '../../lib/parser/WorkSpace';
export interface PackageResult {
packages: Package[];
}
-export const isFileSupported = (filename: string) =>
- isHTMLFile(filename) ??
- isMarkdownFile(filename) ??
- isPlainText(filename) ??
- isCSVFile(filename);
-
-const getPackagesFromZip = async (
- fileContents: Body | undefined,
- paying: boolean,
- settings: Settings
-): Promise => {
- const zipHandler = new ZipHandler();
- const packages = [];
-
- if (!fileContents) {
- return { packages: [] };
- }
-
- zipHandler.build(fileContents as Uint8Array, paying);
-
- const fileNames = zipHandler.getFileNames();
-
- let cardCount = 0;
- for (const fileName of fileNames) {
- if (isFileSupported(fileName)) {
- const deck = await PrepareDeck(fileName, zipHandler.files, settings);
-
- if (deck) {
- packages.push(new Package(deck.name, deck.apkg));
- cardCount += deck.deck.reduce((acc, d) => acc + d.cards.length, 0);
-
- // Checking the limit in place while iterating through the decks
- checkFlashcardsLimits({
- cards: 0,
- decks: deck.deck,
- paying,
- });
- }
- }
-
- // Checking the limit in place while iterating through the files
- checkFlashcardsLimits({
- cards: cardCount,
- paying: paying,
- });
- }
-
- return { packages };
-};
-
class GeneratePackagesUseCase {
- async execute(
+ execute(
paying: boolean,
files: UploadedFile[],
- settings: Settings
+ settings: Settings,
+ workspace: Workspace
): Promise {
- let packages: Package[] = [];
+ return new Promise((resolve, reject) => {
+ const data = { paying, files, settings, workspace };
+ const workerPath = path.resolve(__dirname, './worker.js');
+ const worker = new Worker(workerPath, { workerData: { data } });
- for (const file of files) {
- const fileContents = file.path ? fs.readFileSync(file.path) : file.buffer;
- const filename = file.originalname;
- const key = file.key;
-
- if (isFileSupported(filename)) {
- const d = await PrepareDeck(
- filename,
- [{ name: filename, contents: fileContents }],
- settings
- );
- if (d) {
- const pkg = new Package(d.name, d.apkg);
- packages = packages.concat(pkg);
- }
- } else if (isZIPFile(filename) || isZIPFile(key)) {
- const { packages: extraPackages } = await getPackagesFromZip(
- fileContents,
- paying,
- settings
- );
- packages = packages.concat(extraPackages);
- }
- }
- return { packages };
+ worker.on('message', (result: PackageResult) => resolve(result));
+ worker.on('error', (error) => reject(error));
+ });
}
}
diff --git a/src/usecases/uploads/getPackagesFromZip.ts b/src/usecases/uploads/getPackagesFromZip.ts
new file mode 100644
index 000000000..d0adb715d
--- /dev/null
+++ b/src/usecases/uploads/getPackagesFromZip.ts
@@ -0,0 +1,71 @@
+import { Body } from 'aws-sdk/clients/s3';
+import Settings from '../../lib/parser/Settings';
+import { ZipHandler } from '../../lib/anki/zip';
+import { PrepareDeck } from '../../lib/parser/PrepareDeck';
+import Package from '../../lib/parser/Package';
+import { checkFlashcardsLimits } from '../../lib/User/checkFlashcardsLimits';
+import { PackageResult } from './GeneratePackagesUseCase';
+import {
+ isCSVFile,
+ isHTMLFile,
+ isMarkdownFile,
+ isPlainText,
+} from '../../lib/storage/checks';
+import Workspace from '../../lib/parser/WorkSpace';
+
+export const isFileSupported = (filename: string) =>
+ isHTMLFile(filename) ??
+ isMarkdownFile(filename) ??
+ isPlainText(filename) ??
+ isCSVFile(filename);
+
+export const getPackagesFromZip = async (
+ fileContents: Body | undefined,
+ paying: boolean,
+ settings: Settings,
+ workspace: Workspace
+): Promise => {
+ const zipHandler = new ZipHandler();
+ const packages = [];
+
+ if (!fileContents) {
+ return { packages: [] };
+ }
+
+ zipHandler.build(fileContents as Uint8Array, paying);
+
+ const fileNames = zipHandler.getFileNames();
+
+ let cardCount = 0;
+ for (const fileName of fileNames) {
+ if (isFileSupported(fileName)) {
+ const deck = await PrepareDeck({
+ name: fileName,
+ files: zipHandler.files,
+ settings,
+ noLimits: paying,
+ workspace,
+ });
+
+ if (deck) {
+ packages.push(new Package(deck.name));
+ cardCount += deck.deck.reduce((acc, d) => acc + d.cards.length, 0);
+
+ // Checking the limit in place while iterating through the decks
+ checkFlashcardsLimits({
+ cards: 0,
+ decks: deck.deck,
+ paying,
+ });
+ }
+ }
+
+ // Checking the limit in place while iterating through the files
+ checkFlashcardsLimits({
+ cards: cardCount,
+ paying: paying,
+ });
+ }
+
+ return { packages };
+};
diff --git a/src/usecases/uploads/worker.ts b/src/usecases/uploads/worker.ts
new file mode 100644
index 000000000..516e0d68f
--- /dev/null
+++ b/src/usecases/uploads/worker.ts
@@ -0,0 +1,60 @@
+import { parentPort, workerData } from 'worker_threads';
+import { UploadedFile } from '../../lib/storage/types';
+import Settings from '../../lib/parser/Settings';
+import Package from '../../lib/parser/Package';
+import fs from 'fs';
+import { PrepareDeck } from '../../lib/parser/PrepareDeck';
+import { isZIPFile } from '../../lib/storage/checks';
+import { getPackagesFromZip, isFileSupported } from './getPackagesFromZip';
+import Workspace from '../../lib/parser/WorkSpace';
+
+interface GenerationData {
+ paying: boolean;
+ files: UploadedFile[];
+ settings: Settings;
+ workspace: Workspace;
+}
+
+function doGenerationWork(data: GenerationData) {
+ console.log('doGenerationWork');
+ return new Promise(async (resolve) => {
+ console.log('starting generation');
+ const { paying, files, settings, workspace } = data;
+ let packages: Package[] = [];
+
+ for (const file of files) {
+ const fileContents = file.path ? fs.readFileSync(file.path) : file.buffer;
+ const filename = file.originalname;
+ const key = file.key;
+
+ if (isFileSupported(filename)) {
+ const d = await PrepareDeck({
+ name: filename,
+ files: [{ name: filename, contents: fileContents }],
+ settings,
+ noLimits: paying,
+ workspace,
+ });
+ if (d) {
+ const pkg = new Package(d.name);
+ packages = packages.concat(pkg);
+ }
+ } else if (isZIPFile(filename) || isZIPFile(key)) {
+ const { packages: extraPackages } = await getPackagesFromZip(
+ fileContents,
+ paying,
+ settings,
+ workspace
+ );
+ packages = packages.concat(extraPackages);
+ }
+ }
+ resolve({ packages });
+ });
+}
+
+doGenerationWork(workerData.data)
+ .then((result) => {
+ parentPort?.postMessage(result);
+ })
+ .catch(parentPort?.postMessage);