From e2d6f88e3f720ae49734849ee6ef6effb1bd063b Mon Sep 17 00:00:00 2001 From: Yan Date: Mon, 20 Mar 2023 16:49:04 +0100 Subject: [PATCH 1/3] chore: draft --- CHANGELOG.md | 43 ++++ README.md | 18 +- package-lock.json | 23 ++- package.json | 12 +- src/cli.ts | 19 +- src/comic-downloader.ts | 331 +++++++++++++++++++++++++++++ src/commands.ts | 50 ++++- src/global.d.ts | 10 +- src/index.ts | 424 +------------------------------------- src/lib/index.ts | 5 + src/modules/dmzj/index.ts | 12 ++ src/modules/index.ts | 1 + src/modules/manhuaren.ts | 42 ++++ src/modules/zerobyw.ts | 119 +++++++++++ tsconfig.json | 2 +- 15 files changed, 659 insertions(+), 452 deletions(-) create mode 100644 src/comic-downloader.ts create mode 100644 src/modules/dmzj/index.ts create mode 100644 src/modules/index.ts create mode 100644 src/modules/manhuaren.ts create mode 100644 src/modules/zerobyw.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3120b98..a985381 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,48 @@ # Changelog +## 2.0.0 + +`2023-03-20` + +**`zerobyw-dl` now becomes `comic-dl`.** Now this library is for generic uses. + +### Changes + +- [library] The code has been refactored and can now add sites as plugins. + +```typescript +// Before +import ZeroBywDownloader from "zerobyw-dl"; +const downloader = new ZeroBywDownloader(destination, configs); + +// Now +import { ZeroBywDownloader } from "comic-dl"; +const downloader = new ZeroBywDownloader(destination, configs); +``` + +- [Library] Writing ComicInfo.xml to file is removed from `getSerieInfo`, thus a seperate function `writeComicInfo` has taken place, options from `getSerieInfo` is moved to `writeComicinfo`, and the typedef is renamed `WriteInfoOptions`. + +```typescript +// Before +const serie = await downloader.getSerieInfo("url", { output: true }); + +// Now +const serie = await downloader.getSerieInfo("url"); +await writeComicInfo(serie, { output: true }); +``` + +- [CLI] New flag `-m, --module` is added to specify the module (site) to use, as a matter of which, the short-hand flag to `--max-title-length` is changed to `-M`. + +```bash +# Before +npx zerobyw-dl dl -c cookie.txt -o ~/Download/zerobyw -a zip -r -i -u serie_url + +# Now +npx comic-dl dl -m zerobyw -c cookie.txt -o ~/Download/zerobyw -a zip -r -i -u serie_url +``` + +- [CLI] the `-s, --silence` flag now skips the confirm prompt when downloading series. + ## 1.5.1 `2023-03-18` diff --git a/README.md b/README.md index 9df8bc2..5538319 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,27 @@ -# zerobyw-dl +# comic-dl -[![npm](https://img.shields.io/npm/v/zerobyw-dl.svg)](https://www.npmjs.com/package/zerobyw-dl) -![license](https://img.shields.io/npm/l/zerobyw-dl.svg) -![size](https://img.shields.io/github/repo-size/yinyanfr/zerobyw-dl) +[![npm](https://img.shields.io/npm/v/comic-dl.svg)](https://www.npmjs.com/package/comic-dl) +![license](https://img.shields.io/npm/l/comic-dl.svg) +![size](https://img.shields.io/github/repo-size/yinyanfr/comic-dl) -Yet another batch downloader for [zerobyw](https://zerobyw.github.io/). +As of the version 2, **`zerobyw-dl` now becomes `comic-dl`.** Now this library is for generic uses. + +Looking for `zero-byw`? [Check here](https://github.com/yinyanfr/comic-dl/tree/v1). This library is not for browsers. ## :star2: Features - CLI tools -- Chapter list +- Supports multiple sites (More on the road). - Download as ZIP/CBZ, or just a folder of pictures - Downloading progress watch - Generates [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro) +## Site List + +- [Zerobyw](https://zerobyw.github.io/) + ## :framed_picture: Gallery ### List of chapters diff --git a/package-lock.json b/package-lock.json index 611a133..5fe153e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,26 +1,28 @@ { - "name": "zerobyw-dl", - "version": "1.5.0", + "name": "comic-dl", + "version": "0.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "zerobyw-dl", - "version": "1.5.0", + "name": "comic-dl", + "version": "0.0.2", "license": "MIT", "dependencies": { "archiver": "^5.3.1", "args": "^5.0.3", "axios": "^1.3.4", + "crypto-js": "^4.1.1", "jsdom": "^21.1.1", "yesno": "^0.4.0" }, "bin": { - "zerobyw-dl": "dist/cli.js" + "comic-dl": "dist/cli.js" }, "devDependencies": { "@types/archiver": "^5.3.2", "@types/args": "^5.0.0", + "@types/crypto-js": "^4.1.1", "@types/jsdom": "^21.1.0", "@types/node": "^18.15.0", "typescript": "^4.9.5" @@ -49,6 +51,12 @@ "integrity": "sha512-3fNb8ja/wQWFrHf5SQC5S3n0iBXdnT3PTPEJni2tBQRuv0BnAsz5u12U5gPRBSR7xdY6fI6QjWoTK/8ysuTt0w==", "dev": true }, + "node_modules/@types/crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-BG7fQKZ689HIoc5h+6D2Dgq1fABRa0RbBWKBd9SP/MVRVXROflpm5fhwyATX5duFmbStzyzyycPB8qUYKDH3NA==", + "dev": true + }, "node_modules/@types/jsdom": { "version": "21.1.0", "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.0.tgz", @@ -400,6 +408,11 @@ "node": ">= 10" } }, + "node_modules/crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" + }, "node_modules/cssstyle": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-3.0.0.tgz", diff --git a/package.json b/package.json index e116e72..cbe17ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "zerobyw-dl", - "version": "1.5.1", + "name": "comic-dl", + "version": "0.0.1", "description": "Yet another batch downloader for zerobyw", "main": "dist/index.js", "bin": "dist/cli.js", @@ -17,17 +17,18 @@ ], "repository": { "type": "git", - "url": "git+https://github.com/yinyanfr/zerobyw-dl.git" + "url": "git+https://github.com/yinyanfr/comic-dl.git" }, "bugs": { - "url": "https://github.com/yinyanfr/zerobyw-dl/issues" + "url": "https://github.com/yinyanfr/comic-dl/issues" }, - "homepage": "https://github.com/yinyanfr/zerobyw-dl#readme", + "homepage": "https://github.com/yinyanfr/comic-dl#readme", "author": "Yan", "license": "MIT", "devDependencies": { "@types/archiver": "^5.3.2", "@types/args": "^5.0.0", + "@types/crypto-js": "^4.1.1", "@types/jsdom": "^21.1.0", "@types/node": "^18.15.0", "typescript": "^4.9.5" @@ -36,6 +37,7 @@ "archiver": "^5.3.1", "args": "^5.0.3", "axios": "^1.3.4", + "crypto-js": "^4.1.1", "jsdom": "^21.1.1", "yesno": "^0.4.0" } diff --git a/src/cli.ts b/src/cli.ts index ccb0712..539d47c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,5 +1,10 @@ #!/usr/bin/env node +/** + * MIT License + * Copyright (c) 2023 Yan + */ + import args from "args"; import { chapterCommand, downloadCommand, listCommand } from "./commands"; @@ -18,6 +23,7 @@ args "c", "ch", ]) + .option("module", "Specify the module (site) name.") .option("url", "The url to the serie or the chapter.") .option( "name", @@ -41,7 +47,10 @@ args ) .option("archive", "Optional: Output zip or cbz archive grouped by chapters.") .option("timeout", "Optional: Override the default 10s request timeout.") - .option("slience", "Optional: Silence the console output.") + .option( + "slience", + "Optional: Silence the console output, including the confirm prompt." + ) .option( "batch", "Optional: Set the number or images to be downloaded simultaneously, default to 10." @@ -69,19 +78,19 @@ args ) .option("info", "Optional: Generate ComicInfo.xml.") .example( - "npx zerobyw-dl dl -c cookie.txt -f 10 -t 20 -o ~/Download/zerobyw -a zip -r -i -u serie_url", + "npx comic-dl dl -c cookie.txt -f 10 -t 20 -o ~/Download/manga -a zip -r -i -u serie_url", "Download a serie from its 10th chapter to 20th chapter to the given destination, output zip archives with ComicInfo.xml by chapter, retry if a chapter is not properly downloaded." ) .example( - "npx zerobyw-dl dl -c cookie.txt -o ~/Download/zerobyw -i -u serie_url -c 0,4,12", + "npx comic-dl dl -c cookie.txt -o ~/Download/manga -i -u serie_url -c 0,4,12", "Download chapter index 0, 4, 12 from a serie" ) .example( - "npx zerobyw-dl ls -u serie_url", + "npx comic-dl ls -u serie_url", "List all chapters of the given serie." ) .example( - "npx zerobyw-dl ch -n Chapter1 -u chapter_url -c cookie.txt", + "npx comic-dl ch -n Chapter1 -u chapter_url -c cookie.txt", "Download a chapter named Chapter1 to current path." ); diff --git a/src/comic-downloader.ts b/src/comic-downloader.ts new file mode 100644 index 0000000..cf19bfa --- /dev/null +++ b/src/comic-downloader.ts @@ -0,0 +1,331 @@ +/** + * MIT License + * Copyright (c) 2023 Yan + */ + +import axios from "axios"; +import type { AxiosInstance } from "axios"; +import fs from "node:fs"; +import path from "node:path"; +import type { ReadStream } from "node:fs"; +import archiver from "archiver"; +import type { Archiver } from "archiver"; +import yesno from "yesno"; +import { isString } from "./lib"; + +const ComicInfoFilename = "ComicInfo.xml"; + +export default abstract class ComicDownloader { + static readonly siteName: string; + + protected axios: AxiosInstance; + + constructor(protected destination: string, protected configs: Configs = {}) { + this.axios = axios.create({ + timeout: configs.timeout ?? 10000, + headers: { + Cookie: configs.cookie, + ...(configs.headers || {}), + }, + }); + } + + /** + * ------------------------------------------------- + * For children + */ + + /** + * Get title and chapter list + * @param url Series Url + * @returns Promise, title and list of chapters with array index, name and url + */ + abstract getSerieInfo(url: string): Promise; + + /** + * + * @param url + * @returns list of string or null + */ + protected abstract getImageList(url: string): Promise<(string | null)[]>; + + /** + * End for children + * ------------------------------------------------- + */ + + protected log(content: string) { + if (this.configs.verbose || !this.configs.silence) { + console.log(content); + } + } + + protected generateComicInfoXMLString(info: ComicInfo) { + // TODO: find a reliable module for possible more complicated structures + const identifier = ``; + const open = ``; + const close = ""; + const content = Object.keys(info) + .map((key) => `\t<${key}>${info[key]?.replace?.("\n", "")}`) + .join("\n"); + return `${identifier}\n${open}\n${content}\n${close}\n`; + } + + protected detectBaseUrl(url: string) { + const match = url.match(/^https?:\/\/[^.]+\.[^/]+/); + if (match?.[0]) { + this.axios.defaults.baseURL = match?.[0]; + } + } + + setConfig(key: keyof typeof this.configs, value: any) { + this.configs[key] = value; + } + + setConfigs(configs: Record) { + // Will merge + this.configs = { ...this.configs, ...configs }; + } + + protected async writeComicInfo(serie: SerieInfo, options: WriteInfoOptions) { + if (serie.info) { + const xmlString = this.generateComicInfoXMLString(serie.info); + const chapterPath = path.join( + isString(options.output) + ? (options.output as string) + : this.destination, + options.rename ?? serie.title + ); + if (!fs.existsSync(chapterPath)) { + fs.mkdirSync(chapterPath, { recursive: true }); + } + const writePath = path.join( + chapterPath, + options.filename ?? ComicInfoFilename + ); + await fs.promises.writeFile(writePath, xmlString); + this.log(`Written: ${writePath}`); + } + } + + protected async downloadImage( + chapterName: string, + imageUri: string, + options: ImageDownloadOptions = {} + ) { + const res = await this.axios.get(imageUri, { + responseType: "stream", + }); + if (!res?.data) { + throw new Error("Image Request Failed"); + } + const filenameMatch = imageUri.match(/[^./]+\.[^.]+$/); + const filename = options?.imageName ?? filenameMatch?.[0]; + + if (filename) { + if (options?.archive) { + options.archive.append(res.data, { name: filename }); + return Promise.resolve(); + } else { + const writePath = path.join( + this.destination, + options?.title ? "Untitled" : ".", + chapterName, + filename + ); + const writer = fs.createWriteStream(writePath); + res.data.pipe(writer); + return new Promise((resolve, reject) => { + writer.on("finish", resolve); + writer.on("error", (err) => { + if (this.configs?.verbose) { + console.error(err); + } + reject(); + }); + }); + } + } else { + throw new Error("Cannot Detect Filename."); + } + } + + protected async downloadSegment( + name: string, + segment: (string | null)[], + title?: string, + archive?: Archiver + ) { + const reqs = segment.map((e) => + e ? this.downloadImage(name, e, { title, archive }) : Promise.reject() + ); + const res = await Promise.allSettled(reqs); + return res.filter((e) => e.status === "rejected"); + } + + /** + * Download and write all images from a chapter + * @param name Chapter name + * @param uri Chapter uri + * @param options title, index, onProgress + * @returns DownloadProgress + */ + async downloadChapter( + name: string, + uri?: string, + options: ChapterDownloadOptions = {} + ) { + if (!uri) { + options?.onProgress?.({ + index: options?.index, + name, + uri, + status: "failed", + }); + throw new Error("Invalid Chapter Uri"); + } + if (!this.axios.defaults.baseURL) { + this.detectBaseUrl(uri); + } + const imgList = await this.getImageList(uri); + if (!imgList?.length) { + options?.onProgress?.({ + index: options?.index, + name, + uri, + status: "failed", + }); + throw new Error("Cannot get image list."); + } + const chapterWritePath = path.join( + this.destination, + options?.title ? options.title : ".", + this.configs?.archive ? "." : name + ); + if (!fs.existsSync(chapterWritePath)) { + fs.mkdirSync(chapterWritePath, { recursive: true }); + } + const archive = this.configs?.archive + ? archiver("zip", { zlib: { level: this.configs?.zipLevel ?? 5 } }) + : undefined; + + if (this.configs?.archive) { + const archiveStream = fs.createWriteStream( + path.join( + chapterWritePath, + `${name}.${this.configs?.archive === "cbz" ? "cbz" : "zip"}` + ) + ); + archive?.pipe(archiveStream); + } + + let failures = 0; + const step = this.configs?.batchSize ?? 10; + for (let i = 0; i < imgList.length; i += step) { + const failed = await this.downloadSegment( + name, + imgList.slice(i, Math.min(i + step, imgList.length)), + options?.title, + archive + ); + if (failed?.length) { + failures += failed.length; + this.log( + `Failed: Chapter ${name} - ${failed.length} images not downloaded` + ); + } + } + + if (options.info) { + const xmlString = this.generateComicInfoXMLString(options.info); + if (archive) { + archive.append(xmlString, { name: ComicInfoFilename }); + } else { + await fs.promises.writeFile( + path.join(chapterWritePath, ComicInfoFilename), + xmlString + ); + } + } + + archive?.finalize(); + this.log(`Saved Chapter: [${options?.index}] ${name}`); + const progress = { + index: options?.index, + name, + status: "completed" as const, + failed: failures, + }; + options?.onProgress?.(progress); + + return progress; + } + + /** + * Download from a serie + * @param url serie url + * @param options start, end, confirm, onProgress + */ + async downloadSerie(url: string, options: SerieDownloadOptions = {}) { + this.detectBaseUrl(url); + const serie = await this.getSerieInfo(url); + + if (options?.confirm || !this.configs.silence) { + let queue = "the entire serie"; + if (options.start || options.end) { + queue = `from ${options.start ?? 0} to ${options.end || "the end"}`; + } + if (options.chapters?.length) { + queue = `chapters ${options.chapters?.join(", ")}`; + } + const ok = await yesno({ + question: `Downloading ${serie.title} to ${this.destination}, ${queue}, Proceed? (Y/n)`, + defaultValue: true, + }); + if (!ok) { + console.log("Abort."); + return 0; + } + } + + this.log("Start Downloading..."); + const summary: DownloadProgress[] = []; + const start = options.start ?? 0; + const end = + options.end !== undefined ? options.end + 1 : serie.chapters.length; + + for (let i = start; i < end; i++) { + const chapter = serie.chapters[i]; + if (!options.chapters || options.chapters?.includes(i)) { + const progress = await this.downloadChapter(chapter.name, chapter.uri, { + index: chapter.index, + title: options.rename ?? serie.title, + info: options.info ? serie.info : undefined, + onProgress: options?.onProgress, + }); + summary.push(progress); + } + } + + const failed = summary.filter((e) => e.failed); + if (failed.length) { + this.log(`Download completed with failures.`); + failed.forEach((e) => { + this.log(`Index: ${e.index}, pages not downloaded: ${e.failed}.`); + }); + if (options.retry) { + this.log("Retrying..."); + for (const chapter of failed) { + await this.downloadChapter(chapter.name, chapter.uri, { + index: chapter.index, + title: options.rename ?? serie.title, + info: options.info ? serie.info : undefined, + onProgress: options?.onProgress, + }); + } + } + } else { + this.log("Download Success."); + } + } +} diff --git a/src/commands.ts b/src/commands.ts index 29e44f1..d2a0092 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -1,11 +1,27 @@ #!/usr/bin/env node +/** + * MIT License + * Copyright (c) 2023 Yan + */ + import fs from "node:fs"; import path from "node:path"; -import ZeroBywDownloader from "."; +import * as plugins from "./modules"; + +function findModule(name: string) { + const downloaders = Object.values(plugins); + for (let j = 0; j < downloaders.length; j++) { + const Downloader = downloaders[j]; + if (Downloader.siteName === name) { + return Downloader; + } + } +} function buildDownloader(options: Partial = {}) { const { + module, output, cookie, archive, @@ -16,8 +32,15 @@ function buildDownloader(options: Partial = {}) { maxTitleLength, zipLevel, } = options; + if (!module?.length) { + throw "Please specify module name, i.e. zerobyw"; + } + const Downloader = findModule(module); + if (!Downloader) { + throw "This module is not found."; + } - const downloader = new ZeroBywDownloader(output ?? ".", { + const downloader = new Downloader(output ?? ".", { cookie: cookie && fs.readFileSync(path.resolve(cookie)).toString(), timeout, silence, @@ -33,11 +56,11 @@ function buildDownloader(options: Partial = {}) { export const listCommand: Command = async (name, sub, options = {}) => { const { url, verbose, silence, output, name: rename } = options; - const downloader = buildDownloader(options); try { if (url) { - const serie = await downloader.getSerieInfo(url, { output, rename }); + const downloader = buildDownloader(options); + const serie = await downloader.getSerieInfo(url); if (serie) { if (!silence) { console.log(`Title: ${serie.title}`); @@ -48,10 +71,16 @@ export const listCommand: Command = async (name, sub, options = {}) => { console.log("----"); }); - Object.keys(serie.info).forEach((e) => { - console.log(`${e}: ${serie.info[e]}`); - }); + if (serie?.info) { + Object.keys(serie.info).forEach((e) => { + console.log(`${e}: ${serie.info?.[e]}`); + }); + } } + + // if(output) { + // await downloader.writeCominInfo(serie, { output, rename }) + // } } else { console.log("Please Provide URL."); } @@ -79,11 +108,12 @@ export const downloadCommand: Command = async (name, sub, options = {}) => { chapters, info, } = options; - const downloader = buildDownloader(options); + let current: Partial = {}; try { if (url) { + const downloader = buildDownloader(options); await downloader.downloadSerie(url, { start: from, end: to, @@ -119,7 +149,7 @@ export const downloadCommand: Command = async (name, sub, options = {}) => { ); } else { console.log( - "No chapter is downloaded, please check the availabiliy of zerobyw or your Internet connection." + "No chapter is downloaded, please check the availabiliy of the module (site) or your Internet connection." ); } } @@ -127,10 +157,10 @@ export const downloadCommand: Command = async (name, sub, options = {}) => { export const chapterCommand: Command = async (name, sub, options = {}) => { const { url, name: chapterName, verbose, output } = options; - const downloader = buildDownloader(options); try { if (url) { + const downloader = buildDownloader(options); let serie: SerieInfo | undefined; if (output) { serie = await downloader.getSerieInfo(url); diff --git a/src/global.d.ts b/src/global.d.ts index e91b8ec..e8054e7 100644 --- a/src/global.d.ts +++ b/src/global.d.ts @@ -1,3 +1,8 @@ +/** + * MIT License + * Copyright (c) 2023 Yan + */ + interface Configs { cookie?: string; timeout?: number; @@ -40,7 +45,7 @@ interface ComicInfo { [Key: string]: any; // and many more } -interface SerieInfoOptions { +interface WriteInfoOptions { output?: boolean | string; rename?: string; filename?: string; @@ -52,7 +57,7 @@ interface SerieInfoOptions { interface SerieInfo { title: string; chapters: Chapter[]; - info: ComicInfo; + info?: ComicInfo; } interface DownloadProgress { @@ -88,6 +93,7 @@ interface ChapterDownloadOptions { } interface CliOptions { + module: string; url: string; name: string; output: string; diff --git a/src/index.ts b/src/index.ts index b11035d..28c59ac 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,420 +1,8 @@ -import axios from "axios"; -import type { AxiosInstance } from "axios"; -import { JSDOM } from "jsdom"; -import fs from "node:fs"; -import path from "node:path"; -import type { ReadStream } from "node:fs"; -import archiver from "archiver"; -import type { Archiver } from "archiver"; -import yesno from "yesno"; -import { isString } from "./lib"; +/** + * MIT License + * Copyright (c) 2023 Yan + */ -const Selectors = { - chapters: ".uk-grid-collapse .muludiv a", - page: ".wp", - auth: ".jameson_manhua", - images: ".uk-zjimg img", - tags1: "div.cl > a.uk-label", - tags2: "div.cl > span.uk-label", -}; +export * from "./modules"; -const ComicInfoFilename = "ComicInfo.xml"; - -export default class ZeroBywDownloader { - private axios: AxiosInstance; - - constructor(private destination: string, private configs: Configs = {}) { - this.axios = axios.create({ - timeout: configs.timeout ?? 10000, - headers: { - Cookie: configs.cookie, - ...(configs.headers || {}), - }, - }); - } - - private log(content: string) { - if (this.configs.verbose || !this.configs.silence) { - console.log(content); - } - } - - private generateComicInfoXMLString(info: ComicInfo) { - // the xml module somehow doesn't work as intended - // TODO: find a reliable module for possible more complicated structures - const identifier = ``; - const open = ``; - const close = ""; - const content = Object.keys(info) - .map((key) => `\t<${key}>${info[key]?.replace?.("\n", "")}`) - .join("\n"); - return `${identifier}\n${open}\n${content}\n${close}\n`; - } - - private detectBaseUrl(url: string) { - const match = url.match(/^https?:\/\/[^.]+\.[^/]+/); - if (match?.[0]) { - this.axios.defaults.baseURL = match?.[0]; - } - } - - setConfig(key: keyof typeof this.configs, value: any) { - this.configs[key] = value; - } - - setConfigs(configs: Record) { - // Will merge - this.configs = { ...this.configs, ...configs }; - } - - /** - * Get title and chapter list - * @param url Series Url - * @returns Promise, title and list of chapters with array index, name and url - */ - async getSerieInfo( - url: string, - options: SerieInfoOptions = {} - ): Promise { - const res = await this.axios.get(url); - if (!res?.data) throw new Error("Request Failed."); - - const dom = new JSDOM(res.data); - const document = dom.window.document; - const info: ComicInfo = { Manga: "YesAndRightToLeft", Web: url }; - - const title = - document.querySelector("title")?.textContent?.replace(/[ \s]+/g, "") ?? - "Untitled"; - info.Serie = title; - this.log(`Found ${title}.`); - - const chapterElements = document.querySelectorAll( - Selectors.chapters - ); - - const chapters: Chapter[] = []; - chapterElements.forEach((e, i) => { - chapters.push({ - index: i, - name: e.textContent ?? `${i}`, - uri: (e.getAttribute("href") as string) ?? undefined, - }); - }); - info.Count = chapters.length; - this.log(`Chapter Length: ${chapters.length}`); - - const tagGroup1 = document.querySelectorAll( - Selectors.tags1 - ); - const tags: string[] = []; - tagGroup1?.forEach((tag, i) => { - if (i == 0) { - // plugin.php?id=jameson_manhua&a=zz&zuozhe_name=陈某 - const match = tag.getAttribute("href")?.match(/zuozhe_name=(.+)$/); - if (match) { - info.Penciller = match[1]; - } else { - if (tag.textContent) { - tags.push(tag.textContent); - } - } - } - }); - if (tags.length) { - info.Tags = tags.join(","); - } - - const tagGroup2 = document.querySelectorAll( - Selectors.tags2 - ); - const lang = tagGroup2[0]?.textContent; - info.Language = lang === "全生肉" ? "jp" : "zh"; - info.Location = tagGroup2[1]?.textContent ?? undefined; - const status = tagGroup2[2]?.textContent; - if (status === "连载中") { - info.Status = "Ongoing"; - } - if (status === "已完结") { - info.Status = "End"; - } - - const summary = document.querySelector("li > div.uk-alert"); - info.Summary = summary?.textContent ?? undefined; - - const serieInfo: SerieInfo = { - title: this.configs?.maxTitleLength - ? title.slice(0, this.configs?.maxTitleLength) - : title, - chapters, - info, - }; - if (options.output) { - const xmlString = this.generateComicInfoXMLString(info); - const chapterPath = path.join( - isString(options.output) - ? (options.output as string) - : this.destination, - options.rename ?? title - ); - if (!fs.existsSync(chapterPath)) { - fs.mkdirSync(chapterPath, { recursive: true }); - } - const writePath = path.join( - chapterPath, - options.filename ?? ComicInfoFilename - ); - await fs.promises.writeFile(writePath, xmlString); - this.log(`Written: ${writePath}`); - } - - return serieInfo; - } - - private async getImageList(url: string) { - const res = await this.axios.get(url); - if (!res?.data) throw new Error("Request Failed."); - - const dom = new JSDOM(res.data); - const document = dom.window.document; - if (!document.querySelectorAll(Selectors.page)?.length) { - throw new Error("Invalid Page."); - } - if (!document.querySelectorAll(Selectors.auth)?.length) { - throw new Error("Unauthorized: Please log in."); - } - const imageElements = document.querySelectorAll( - Selectors.images - ); - if (!imageElements?.length) { - throw new Error("Forbidden: This chapter requires a VIP user rank."); - } - - const imageList: (string | null)[] = []; - imageElements.forEach((e) => { - imageList.push(e.getAttribute("src")); - }); - return imageList; - } - - private async downloadImage( - chapterName: string, - imageUri: string, - options: ImageDownloadOptions = {} - ) { - const res = await this.axios.get(imageUri, { - responseType: "stream", - }); - if (!res?.data) { - throw new Error("Image Request Failed"); - } - const filenameMatch = imageUri.match(/[^./]+\.[^.]+$/); - const filename = options?.imageName ?? filenameMatch?.[0]; - - if (filename) { - if (options?.archive) { - options.archive.append(res.data, { name: filename }); - return Promise.resolve(); - } else { - const writePath = path.join( - this.destination, - options?.title ? "Untitled" : ".", - chapterName, - filename - ); - const writer = fs.createWriteStream(writePath); - res.data.pipe(writer); - return new Promise((resolve, reject) => { - writer.on("finish", resolve); - writer.on("error", (err) => { - if (this.configs?.verbose) { - console.error(err); - } - reject(); - }); - }); - } - } else { - throw new Error("Cannot Detect Filename."); - } - } - - private async downloadSegment( - name: string, - segment: (string | null)[], - title?: string, - archive?: Archiver - ) { - const reqs = segment.map((e) => - e ? this.downloadImage(name, e, { title, archive }) : Promise.reject() - ); - const res = await Promise.allSettled(reqs); - return res.filter((e) => e.status === "rejected"); - } - - /** - * Download and write all images from a chapter - * @param name Chapter name - * @param uri Chapter uri - * @param options title, index, onProgress - * @returns DownloadProgress - */ - async downloadChapter( - name: string, - uri?: string, - options: ChapterDownloadOptions = {} - ) { - if (!uri) { - options?.onProgress?.({ - index: options?.index, - name, - uri, - status: "failed", - }); - throw new Error("Invalid Chapter Uri"); - } - if (!this.axios.defaults.baseURL) { - this.detectBaseUrl(uri); - } - const imgList = await this.getImageList(uri); - if (!imgList?.length) { - options?.onProgress?.({ - index: options?.index, - name, - uri, - status: "failed", - }); - throw new Error("Cannot get image list."); - } - const chapterWritePath = path.join( - this.destination, - options?.title ? options.title : ".", - this.configs?.archive ? "." : name - ); - if (!fs.existsSync(chapterWritePath)) { - fs.mkdirSync(chapterWritePath, { recursive: true }); - } - const archive = this.configs?.archive - ? archiver("zip", { zlib: { level: this.configs?.zipLevel ?? 5 } }) - : undefined; - - if (this.configs?.archive) { - const archiveStream = fs.createWriteStream( - path.join( - chapterWritePath, - `${name}.${this.configs?.archive === "cbz" ? "cbz" : "zip"}` - ) - ); - archive?.pipe(archiveStream); - } - - let failures = 0; - const step = this.configs?.batchSize ?? 10; - for (let i = 0; i < imgList.length; i += step) { - const failed = await this.downloadSegment( - name, - imgList.slice(i, Math.min(i + step, imgList.length)), - options?.title, - archive - ); - if (failed?.length) { - failures += failed.length; - this.log( - `Failed: Chapter ${name} - ${failed.length} images not downloaded` - ); - } - } - - if (options.info) { - const xmlString = this.generateComicInfoXMLString(options.info); - if (archive) { - archive.append(xmlString, { name: ComicInfoFilename }); - } else { - await fs.promises.writeFile( - path.join(chapterWritePath, ComicInfoFilename), - xmlString - ); - } - } - - archive?.finalize(); - this.log(`Saved Chapter: [${options?.index}] ${name}`); - const progress = { - index: options?.index, - name, - status: "completed" as const, - failed: failures, - }; - options?.onProgress?.(progress); - - return progress; - } - - /** - * Download from a serie - * @param url serie url - * @param options start, end, confirm, onProgress - */ - async downloadSerie(url: string, options: SerieDownloadOptions = {}) { - this.detectBaseUrl(url); - const serie = await this.getSerieInfo(url); - - if (options?.confirm) { - let queue = "the entire serie"; - if (options.start || options.end) { - queue = `from ${options.start ?? 0} to ${options.end || "the end"}`; - } - if (options.chapters?.length) { - queue = `chapters ${options.chapters?.join(", ")}`; - } - const ok = await yesno({ - question: `Downloading ${serie.title} to ${this.destination}, ${queue}, Proceed? (Y/n)`, - defaultValue: true, - }); - if (!ok) { - console.log("Abort."); - return 0; - } - } - - this.log("Start Downloading..."); - const summary: DownloadProgress[] = []; - const start = options.start ?? 0; - const end = - options.end !== undefined ? options.end + 1 : serie.chapters.length; - - for (let i = start; i < end; i++) { - const chapter = serie.chapters[i]; - if (!options.chapters || options.chapters?.includes(i)) { - const progress = await this.downloadChapter(chapter.name, chapter.uri, { - index: chapter.index, - title: options.rename ?? serie.title, - info: options.info ? serie.info : undefined, - onProgress: options?.onProgress, - }); - summary.push(progress); - } - } - - const failed = summary.filter((e) => e.failed); - if (failed.length) { - this.log(`Download completed with failures.`); - failed.forEach((e) => { - this.log(`Index: ${e.index}, pages not downloaded: ${e.failed}.`); - }); - if (options.retry) { - this.log("Retrying..."); - for (const chapter of failed) { - await this.downloadChapter(chapter.name, chapter.uri, { - index: chapter.index, - title: options.rename ?? serie.title, - info: options.info ? serie.info : undefined, - onProgress: options?.onProgress, - }); - } - } - } else { - this.log("Download Success."); - } - } -} +export { default as default } from "./comic-downloader"; diff --git a/src/lib/index.ts b/src/lib/index.ts index 042f1b8..4520faf 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -1,3 +1,8 @@ +/** + * MIT License + * Copyright (c) 2023 Yan + */ + export function isString(n: unknown) { return typeof n === "string" || n instanceof String; } diff --git a/src/modules/dmzj/index.ts b/src/modules/dmzj/index.ts new file mode 100644 index 0000000..a801e62 --- /dev/null +++ b/src/modules/dmzj/index.ts @@ -0,0 +1,12 @@ +import ComicDownloader from "../../comic-downloader"; + +export default class DMZJDownloader extends ComicDownloader { + static readonly siteName = "dmzj"; + + getSerieInfo(url: string): Promise { + throw new Error("Method not implemented."); + } + protected getImageList(url: string): Promise<(string | null)[]> { + throw new Error("Method not implemented."); + } +} diff --git a/src/modules/index.ts b/src/modules/index.ts new file mode 100644 index 0000000..38303d9 --- /dev/null +++ b/src/modules/index.ts @@ -0,0 +1 @@ +export { default as ZeroBywDownloader } from "./zerobyw"; diff --git a/src/modules/manhuaren.ts b/src/modules/manhuaren.ts new file mode 100644 index 0000000..0d0edac --- /dev/null +++ b/src/modules/manhuaren.ts @@ -0,0 +1,42 @@ +import MD5 from "crypto-js/md5"; +import { URL } from "url"; +import ComicDownloader from "../comic-downloader"; + +const PrivateKey = "4e0a48e1c0b54041bce9c8f0e036124d"; + +function generateGSNHash(url: string) { + const parsed = new URL(url); + let s = `${PrivateKey}GET`; + parsed.searchParams.forEach((key) => { + if (key !== "gsn") { + s += key; + const value = parsed.searchParams.get(key); + if (value) { + s += encodeURI(value); + } + } + }); + s += PrivateKey; + return MD5(s); +} + +export default class ManhuarenDownloader extends ComicDownloader { + static readonly siteName = "manhuaren"; + static readonly baseUrl = "http://mangaapi.manhuaren.com"; + + constructor(destination: string, configs: Configs) { + super(destination, configs); + this.axios.defaults.headers["X-Yq-Yqci"] = '{"le": "zh"}'; + this.axios.defaults.headers["User-Agent"] = "okhttp/3.11.0"; + this.axios.defaults.headers["Referer"] = "http://www.dm5.com/dm5api/"; + this.axios.defaults.headers["clubReferer"] = + "http://mangaapi.manhuaren.com/"; + } + + getSerieInfo(url: string): Promise { + throw new Error("Method not implemented."); + } + protected getImageList(url: string): Promise<(string | null)[]> { + throw new Error("Method not implemented."); + } +} diff --git a/src/modules/zerobyw.ts b/src/modules/zerobyw.ts new file mode 100644 index 0000000..c941193 --- /dev/null +++ b/src/modules/zerobyw.ts @@ -0,0 +1,119 @@ +import { JSDOM } from "jsdom"; +import ComicDownloader from "../comic-downloader"; + +const Selectors = { + chapters: ".uk-grid-collapse .muludiv a", + page: ".wp", + auth: ".jameson_manhua", + images: ".uk-zjimg img", + tags1: "div.cl > a.uk-label", + tags2: "div.cl > span.uk-label", +}; + +export default class ZeroBywDownloader extends ComicDownloader { + static readonly siteName = "zerobyw"; + + async getSerieInfo(url: string): Promise { + const res = await this.axios.get(url); + if (!res?.data) throw new Error("Request Failed."); + + const dom = new JSDOM(res.data); + const document = dom.window.document; + const info: ComicInfo = { Manga: "YesAndRightToLeft", Web: url }; + + const title = + document.querySelector("title")?.textContent?.replace(/[ \s]+/g, "") ?? + "Untitled"; + info.Serie = title; + this.log(`Found ${title}.`); + + const chapterElements = document.querySelectorAll( + Selectors.chapters + ); + + const chapters: Chapter[] = []; + chapterElements.forEach((e, i) => { + chapters.push({ + index: i, + name: e.textContent ?? `${i}`, + uri: (e.getAttribute("href") as string) ?? undefined, + }); + }); + info.Count = chapters.length; + this.log(`Chapter Length: ${chapters.length}`); + + const tagGroup1 = document.querySelectorAll( + Selectors.tags1 + ); + const tags: string[] = []; + tagGroup1?.forEach((tag, i) => { + if (i == 0) { + // plugin.php?id=jameson_manhua&a=zz&zuozhe_name=陈某 + const match = tag.getAttribute("href")?.match(/zuozhe_name=(.+)$/); + if (match) { + info.Penciller = match[1]; + } else { + if (tag.textContent) { + tags.push(tag.textContent); + } + } + } + }); + if (tags.length) { + info.Tags = tags.join(","); + } + + const tagGroup2 = document.querySelectorAll( + Selectors.tags2 + ); + const lang = tagGroup2[0]?.textContent; + info.Language = lang === "全生肉" ? "jp" : "zh"; + info.Location = tagGroup2[1]?.textContent ?? undefined; + const status = tagGroup2[2]?.textContent; + if (status === "连载中") { + info.Status = "Ongoing"; + } + if (status === "已完结") { + info.Status = "End"; + } + + const summary = document.querySelector("li > div.uk-alert"); + info.Summary = summary?.textContent ?? undefined; + + const serieInfo: SerieInfo = { + title: this.configs?.maxTitleLength + ? title.slice(0, this.configs?.maxTitleLength) + : title, + chapters, + info, + }; + + return serieInfo; + } + + protected async getImageList(url: string) { + const res = await this.axios.get(url); + if (!res?.data) throw new Error("Request Failed."); + + const dom = new JSDOM(res.data); + const document = dom.window.document; + if (!document.querySelectorAll(Selectors.page)?.length) { + throw new Error("Invalid Page."); + } + if (!document.querySelectorAll(Selectors.auth)?.length) { + throw new Error("Unauthorized: Please log in."); + } + const imageElements = document.querySelectorAll( + Selectors.images + ); + if (!imageElements?.length) { + throw new Error("Forbidden: This chapter requires a VIP user rank."); + } + + const imageList: (string | null)[] = []; + imageElements.forEach((e) => { + imageList.push(e.getAttribute("src")); + }); + return imageList; + } +} diff --git a/tsconfig.json b/tsconfig.json index dc8e354..258f751 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,7 +12,7 @@ "target": "ESNext", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ - // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ From d9743885d5b695037e9c8208136bd373ea7f3ea0 Mon Sep 17 00:00:00 2001 From: Yan Date: Fri, 31 Mar 2023 19:55:00 +0200 Subject: [PATCH 2/3] feat: add Copymanga and much more --- .gitignore | 1 + CHANGELOG.md | 21 ++- README.md | 84 +++++---- package-lock.json | 18 +- package.json | 9 +- src/cli.ts | 20 +- src/comic-downloader.ts | 44 +++-- src/commands.ts | 36 ++-- src/global.d.ts | 12 +- src/lib/index.ts | 4 + src/modules/copymanga/index.d.ts | 188 +++++++++++++++++++ src/modules/copymanga/index.ts | 109 +++++++++++ src/modules/dmzj/index.ts | 12 -- src/modules/index.ts | 1 + src/modules/manhuaren.ts | 42 ----- src/modules/{zerobyw.ts => zerobyw/index.ts} | 15 +- 16 files changed, 477 insertions(+), 139 deletions(-) create mode 100644 src/modules/copymanga/index.d.ts create mode 100644 src/modules/copymanga/index.ts delete mode 100644 src/modules/dmzj/index.ts delete mode 100644 src/modules/manhuaren.ts rename src/modules/{zerobyw.ts => zerobyw/index.ts} (91%) diff --git a/.gitignore b/.gitignore index 3beee05..9f72be1 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,4 @@ dist # tmp tmp/ +src/tmp.js diff --git a/CHANGELOG.md b/CHANGELOG.md index a985381..aaf8e1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,20 @@ `2023-03-20` -**`zerobyw-dl` now becomes `comic-dl`.** Now this library is for generic uses. +**`zerobyw-dl` now becomes `comic-dl`.** + +Now this library is designed to be used with multiple manga / comic sites. + +### Site Support + +- Added [Copymanga](https://www.copymanga.site/) ### Changes - [library] The code has been refactored and can now add sites as plugins. +- [CLI] The `-b, --batch` flag is now default to 1 when not set. +- Downloaded images are now renamed by index (01 ~ ). +- Downloaders now ignores downloaded chapters by default, set `configs.override` to `true` or for CLI use `-O, --override` if you want to override. ```typescript // Before @@ -31,18 +40,24 @@ const serie = await downloader.getSerieInfo("url"); await writeComicInfo(serie, { output: true }); ``` -- [CLI] New flag `-m, --module` is added to specify the module (site) to use, as a matter of which, the short-hand flag to `--max-title-length` is changed to `-M`. +- [CLI] New flag `-m, --module` is added to specify the module (site) to use, as a matter of which, the short-hand flag to `--max-title-length` is changed to `-M`. If `--module` is not defined, comic-dl will attempt to detect the matching module by url. ```bash # Before npx zerobyw-dl dl -c cookie.txt -o ~/Download/zerobyw -a zip -r -i -u serie_url # Now -npx comic-dl dl -m zerobyw -c cookie.txt -o ~/Download/zerobyw -a zip -r -i -u serie_url +npx comic-dl dl -m zerobyw -c cookie.txt -o ~/Download/zerobyw -a zip -r -i -u serie_url -b 10 +# You can skip -m flag unless comic-dl fails to detect the site module +# Batch download is now default to 1, set it manually for download speed ``` - [CLI] the `-s, --silence` flag now skips the confirm prompt when downloading series. +### Fix + +- [CLI] Fixed an error that causes the downloader to download the entire serie when `-C, --chapters` is set to `0`. + ## 1.5.1 `2023-03-18` diff --git a/README.md b/README.md index 5538319..8ba218b 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ This library is not for browsers. ## Site List - [Zerobyw](https://zerobyw.github.io/) +- [Copymanga](https://www.copymanga.site/) ## :framed_picture: Gallery @@ -46,45 +47,50 @@ npx zerobyw-dl help ## :wrench: Cli ``` -Usage: zerobyw-dl [options] [command] - -Commands: - chapter, c, ch Download images from one chapter. - download, d, dl Download chapters from a manga serie. - help Display help - list, l, ls List all chapters of a manga serie. - version Display version - -Options: - -a, --archive Optional: Output zip or cbz archive grouped by chapters. - -b, --batch Optional: Set the number or images to be downloaded simultaneously, default to 10. - -C, --chapters Optional: Only downloading given list of chapters, example: -C 1,2,4,7 - -c, --cookie Optional (but recommanded): Provide the path to a text file that contains your cookie. - -f, --from Optional: Starting chapter when downloading a serie, default to 0. - -h, --help Output usage information - -i, --info Optional: Generate ComicInfo.xml. - -m, --max-title-length Optional: restrict the length of title as the folder name. - -n, --name Optional: Proride the serie title and override the folder name. - -o, --output Optional: The path where downloaded files are saved (default to .), setting this flag when using list will save a ComicInfo.xml to the path. - -r, --retry Optional: Automatically re-download chapters with failed images. - -s, --slience Optional: Silence the console output. - -T, --timeout Optional: Override the default 10s request timeout. - -t, --to Optional: Ending chapter when downloading a serie, defaults to chapter.length - 1. - -u, --url The url to the serie or the chapter. - -v, --verbose Optional: Display detailed error message, overrides silence. - -V, --version Output the version number - -y, --yes Optional: Skipping confirmation prompt when downloading series. - -z, --zip-level Optional: zip level for archive, default to 5. - -Examples: - - Download a serie from its 10th chapter to 20th chapter to the given destination, output zip archives with ComicInfo.xml by chapter, retry if a chapter is not properly downloaded. - $ npx zerobyw-dl dl -c cookie.txt -f 10 -t 20 -o ~/Download/zerobyw -a zip -r -i -u serie_url - - - List all chapters of the given serie. - $ npx zerobyw-dl ls -u serie_url - - - Download a chapter named Chapter1 to current path. - $ npx zerobyw-dl ch -n Chapter1 -u chapter_url -c cookie.txt + Usage: comic-dl [options] [command] + + Commands: + chapter, c, ch Download images from one chapter. + download, d, dl Download chapters from a manga serie. + help Display help + list, l, ls List all chapters of a manga serie. + version Display version + + Options: + -a, --archive Optional: Output zip or cbz archive grouped by chapters. + -b, --batch Optional: Set the number or images to be downloaded simultaneously, default to 1. + -C, --chapters Optional: Only downloading given list of chapters, example: -C 1,2,4,7 + -c, --cookie Optional (but recommanded): Provide the path to a text file that contains your cookie. + -F, --format Optional: the format of downloaded picture, depending on the modules, example: webp / jpg. + -f, --from Optional: Starting chapter when downloading a serie, default to 0. + -h, --help Output usage information + -i, --info Optional: Generate ComicInfo.xml. + -M, --max-title-length Optional: restrict the length of title as the folder name. + -m, --module Optional: Specify the module (site) name. Will attempt to detect module by url if not set. + -n, --name Optional: Proride the serie title and override the folder name. + -o, --output Optional: The path where downloaded files are saved (default to .), setting this flag when using list will save a ComicInfo.xml to the path. + -r, --retry Optional: Automatically re-download chapters with failed images. + -s, --slience Optional: Silence the console output, including the confirm prompt. + -T, --timeout Optional: Override the default 10s request timeout. + -t, --to Optional: Ending chapter when downloading a serie, defaults to chapter.length - 1. + -u, --url The url to the serie or the chapter. + -v, --verbose Optional: Display detailed error message, overrides silence. + -V, --version Output the version number + -y, --yes Optional: Skipping confirmation prompt when downloading series. + -z, --zip-level Optional: zip level for archive, default to 5. + + Examples: + - Download a serie from its 10th chapter to 20th chapter to the given destination, 10 images at a time, output zip archives with ComicInfo.xml by chapter, retry if a chapter is not properly downloaded. + $ npx comic-dl dl -c cookie.txt -f 10 -t 20 -o ~/Download/manga -a zip -r -i -b 10 -u serie_url + + - Download chapter index 0, 4, 12 from a serie + $ npx comic-dl dl -c cookie.txt -o ~/Download/manga -i -u serie_url -c 0,4,12 + + - List all chapters of the given serie. + $ npx comic-dl ls -u serie_url + + - Download a chapter named Chapter1 to current path. + $ npx comic-dl ch -n Chapter1 -u chapter_url -c cookie.txt ``` ## :book: Library diff --git a/package-lock.json b/package-lock.json index 5fe153e..3a7458a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,19 +1,21 @@ { "name": "comic-dl", - "version": "0.0.2", + "version": "2.0.0-alpha", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "comic-dl", - "version": "0.0.2", + "version": "2.0.0-alpha", "license": "MIT", "dependencies": { "archiver": "^5.3.1", "args": "^5.0.3", "axios": "^1.3.4", "crypto-js": "^4.1.1", + "dayjs": "^1.11.7", "jsdom": "^21.1.1", + "mime-types": "^2.1.35", "yesno": "^0.4.0" }, "bin": { @@ -24,6 +26,7 @@ "@types/args": "^5.0.0", "@types/crypto-js": "^4.1.1", "@types/jsdom": "^21.1.0", + "@types/mime-types": "^2.1.1", "@types/node": "^18.15.0", "typescript": "^4.9.5" } @@ -68,6 +71,12 @@ "parse5": "^7.0.0" } }, + "node_modules/@types/mime-types": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.1.tgz", + "integrity": "sha512-vXOTGVSLR2jMw440moWTC7H19iUyLtP3Z1YTj7cSsubOICinjMxFeb/V57v9QdyyPGbbWolUFSSmSiRSn94tFw==", + "dev": true + }, "node_modules/@types/node": { "version": "18.15.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.0.tgz", @@ -437,6 +446,11 @@ "node": ">=14" } }, + "node_modules/dayjs": { + "version": "1.11.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz", + "integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==" + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", diff --git a/package.json b/package.json index cbe17ea..2952cc2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "comic-dl", - "version": "0.0.1", - "description": "Yet another batch downloader for zerobyw", + "version": "2.0.0-alpha", + "description": "Yet another batch downloader for manga / comic sites", "main": "dist/index.js", "bin": "dist/cli.js", "types": "src/global.d.ts", @@ -30,6 +30,7 @@ "@types/args": "^5.0.0", "@types/crypto-js": "^4.1.1", "@types/jsdom": "^21.1.0", + "@types/mime-types": "^2.1.1", "@types/node": "^18.15.0", "typescript": "^4.9.5" }, @@ -38,7 +39,9 @@ "args": "^5.0.3", "axios": "^1.3.4", "crypto-js": "^4.1.1", + "dayjs": "^1.11.7", "jsdom": "^21.1.1", + "mime-types": "^2.1.35", "yesno": "^0.4.0" } -} \ No newline at end of file +} diff --git a/src/cli.ts b/src/cli.ts index 539d47c..7499f5e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -23,7 +23,10 @@ args "c", "ch", ]) - .option("module", "Specify the module (site) name.") + .option( + "module", + "Optional: Specify the module (site) name. Will attempt to detect module by url if not set." + ) .option("url", "The url to the serie or the chapter.") .option( "name", @@ -53,7 +56,7 @@ args ) .option( "batch", - "Optional: Set the number or images to be downloaded simultaneously, default to 10." + "Optional: Set the number or images to be downloaded simultaneously, default to 1." ) .option( "verbose", @@ -77,13 +80,18 @@ args "Optional: Only downloading given list of chapters, example: -C 1,2,4,7" ) .option("info", "Optional: Generate ComicInfo.xml.") + .option( + "format", + "Optional: the format of downloaded picture, depending on the modules, example: webp / jpg." + ) + .option("override", "Optional: overrides downloaded chapters.") .example( - "npx comic-dl dl -c cookie.txt -f 10 -t 20 -o ~/Download/manga -a zip -r -i -u serie_url", - "Download a serie from its 10th chapter to 20th chapter to the given destination, output zip archives with ComicInfo.xml by chapter, retry if a chapter is not properly downloaded." + "npx comic-dl dl -c cookie.txt -f 10 -t 20 -o ~/Download/manga -a zip -r -i -b 10 -u serie_url", + "Download a serie from its 10th chapter to 20th chapter to the given destination, 10 images at a time, output zip archives with ComicInfo.xml by chapter, retry if a chapter is not properly downloaded." ) .example( - "npx comic-dl dl -c cookie.txt -o ~/Download/manga -i -u serie_url -c 0,4,12", - "Download chapter index 0, 4, 12 from a serie" + "npx comic-dl dl -c cookie.txt -o ~/Download/manga -i -O -u serie_url -c 0,4,12", + "Download chapter index 0, 4, 12 from a serie, overriding downloaded files." ) .example( "npx comic-dl ls -u serie_url", diff --git a/src/comic-downloader.ts b/src/comic-downloader.ts index cf19bfa..61f986c 100644 --- a/src/comic-downloader.ts +++ b/src/comic-downloader.ts @@ -9,15 +9,19 @@ import fs from "node:fs"; import path from "node:path"; import type { ReadStream } from "node:fs"; import archiver from "archiver"; -import type { Archiver } from "archiver"; import yesno from "yesno"; -import { isString } from "./lib"; +import { formatImageName, isString } from "./lib"; +import mime from "mime-types"; const ComicInfoFilename = "ComicInfo.xml"; export default abstract class ComicDownloader { static readonly siteName: string; + static canHandleUrl(url: string): boolean { + return false; + } + protected axios: AxiosInstance; constructor(protected destination: string, protected configs: Configs = {}) { @@ -71,12 +75,7 @@ export default abstract class ComicDownloader { return `${identifier}\n${open}\n${content}\n${close}\n`; } - protected detectBaseUrl(url: string) { - const match = url.match(/^https?:\/\/[^.]+\.[^/]+/); - if (match?.[0]) { - this.axios.defaults.baseURL = match?.[0]; - } - } + protected detectBaseUrl(url: string): void {} setConfig(key: keyof typeof this.configs, value: any) { this.configs[key] = value; @@ -87,7 +86,7 @@ export default abstract class ComicDownloader { this.configs = { ...this.configs, ...configs }; } - protected async writeComicInfo(serie: SerieInfo, options: WriteInfoOptions) { + async writeComicInfo(serie: SerieInfo, options: WriteInfoOptions) { if (serie.info) { const xmlString = this.generateComicInfoXMLString(serie.info); const chapterPath = path.join( @@ -119,8 +118,11 @@ export default abstract class ComicDownloader { if (!res?.data) { throw new Error("Image Request Failed"); } + const ext = mime.extension(res.headers["content-type"]); const filenameMatch = imageUri.match(/[^./]+\.[^.]+$/); - const filename = options?.imageName ?? filenameMatch?.[0]; + const filename = options?.imageName + ? `${options.imageName}.${ext ?? "jpg"}` + : filenameMatch?.[0]; if (filename) { if (options?.archive) { @@ -153,11 +155,16 @@ export default abstract class ComicDownloader { protected async downloadSegment( name: string, segment: (string | null)[], - title?: string, - archive?: Archiver + options: SegmentDownloadOptions = {} ) { - const reqs = segment.map((e) => - e ? this.downloadImage(name, e, { title, archive }) : Promise.reject() + const reqs = segment.map((e, i) => + e + ? this.downloadImage(name, e, { + title: options.title, + archive: options.archive, + imageName: formatImageName((options.offset ?? 0) + i + 1), + }) + : Promise.reject() ); const res = await Promise.allSettled(reqs); return res.filter((e) => e.status === "rejected"); @@ -225,8 +232,11 @@ export default abstract class ComicDownloader { const failed = await this.downloadSegment( name, imgList.slice(i, Math.min(i + step, imgList.length)), - options?.title, - archive + { + offset: i, + title: options?.title, + archive, + } ); if (failed?.length) { failures += failed.length; @@ -269,6 +279,8 @@ export default abstract class ComicDownloader { async downloadSerie(url: string, options: SerieDownloadOptions = {}) { this.detectBaseUrl(url); const serie = await this.getSerieInfo(url); + this.log(`Found ${serie.title}`); + this.log(`Chapters Count: ${serie.chapters?.length}`); if (options?.confirm || !this.configs.silence) { let queue = "the entire serie"; diff --git a/src/commands.ts b/src/commands.ts index d2a0092..3020fa9 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -9,6 +9,17 @@ import fs from "node:fs"; import path from "node:path"; import * as plugins from "./modules"; +function detectModule(url?: string) { + if (!url) return undefined; + const downloaders = Object.values(plugins); + for (let j = 0; j < downloaders.length; j++) { + const Downloader = downloaders[j]; + if (Downloader.canHandleUrl(url)) { + return Downloader; + } + } +} + function findModule(name: string) { const downloaders = Object.values(plugins); for (let j = 0; j < downloaders.length; j++) { @@ -31,24 +42,24 @@ function buildDownloader(options: Partial = {}) { verbose, maxTitleLength, zipLevel, + url, + format, } = options; - if (!module?.length) { - throw "Please specify module name, i.e. zerobyw"; - } - const Downloader = findModule(module); + const Downloader = module?.length ? findModule(module) : detectModule(url); if (!Downloader) { - throw "This module is not found."; + throw new Error("Module not found."); } const downloader = new Downloader(output ?? ".", { cookie: cookie && fs.readFileSync(path.resolve(cookie)).toString(), timeout, silence, - batchSize: batch, + batchSize: batch ?? 1, verbose, archive, maxTitleLength, zipLevel, + format, }); return downloader; @@ -78,9 +89,9 @@ export const listCommand: Command = async (name, sub, options = {}) => { } } - // if(output) { - // await downloader.writeCominInfo(serie, { output, rename }) - // } + if (output) { + await downloader.writeComicInfo(serie, { output, rename }); + } } else { console.log("Please Provide URL."); } @@ -121,9 +132,10 @@ export const downloadCommand: Command = async (name, sub, options = {}) => { rename, retry, info, - chapters: chapters - ? `${chapters}`.split(",").map((e) => parseInt(e)) - : undefined, + chapters: + chapters !== undefined + ? `${chapters}`.split(",").map((e) => parseInt(e)) + : undefined, }); } else { console.log("Please Provide URL."); diff --git a/src/global.d.ts b/src/global.d.ts index e8054e7..30fe330 100644 --- a/src/global.d.ts +++ b/src/global.d.ts @@ -13,6 +13,8 @@ interface Configs { headers?: Record; maxTitleLength?: number; zipLevel?: number; + format?: string; + override?: boolean; } interface Chapter { @@ -92,6 +94,12 @@ interface ChapterDownloadOptions { onProgress?: (progress: DownloadProgress) => void; } +interface SegmentDownloadOptions { + offset?: number; + title?: string; + archive?: Archiver; +} + interface CliOptions { module: string; url: string; @@ -110,7 +118,9 @@ interface CliOptions { zipLevel: number; // zip-level retry: boolean; chapters: string | number; // 1,2,4,7 as string or a single number - info?: boolean; + info: boolean; + format: string; + override: boolean; } type Command = ( diff --git a/src/lib/index.ts b/src/lib/index.ts index 4520faf..509b560 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -6,3 +6,7 @@ export function isString(n: unknown) { return typeof n === "string" || n instanceof String; } + +export function formatImageName(index: number) { + return index < 10 ? `0${index}` : `${index}`; +} diff --git a/src/modules/copymanga/index.d.ts b/src/modules/copymanga/index.d.ts new file mode 100644 index 0000000..57b8390 --- /dev/null +++ b/src/modules/copymanga/index.d.ts @@ -0,0 +1,188 @@ +declare namespace CopymangaAPI { + declare namespace Serie { + export interface Data { + code: number; + message: string; + results: Results; + } + + export interface Results { + is_lock: boolean; + is_login: boolean; + is_mobile_bind: boolean; + is_vip: boolean; + comic: Comic; + popular: number; + groups: Groups; + } + + export interface Comic { + uuid: string; + b_404: boolean; + b_hidden: boolean; + name: string; + alias: string; + path_word: string; + close_comment: boolean; + close_roast: boolean; + free_type: FreeType; + restrict: Restrict; + reclass: Reclass; + females: any[]; + males: any[]; + clubs: any[]; + img_type: number; + seo_baidu: string; + region: Region; + status: Status; + author: Author[]; + theme: Theme[]; + parodies: any[]; + brief: string; + datetime_updated: string; + cover: string; + last_chapter: LastChapter; + popular: number; + } + + export interface FreeType { + display: string; + value: number; + } + + export interface Restrict { + value: number; + display: string; + } + + export interface Reclass { + value: number; + display: string; + } + + export interface Region { + value: number; + display: string; + } + + export interface Status { + value: number; + display: string; + } + + export interface Author { + name: string; + path_word: string; + } + + export interface Theme { + name: string; + path_word: string; + } + + export interface LastChapter { + uuid: string; + name: string; + } + + export interface Groups { + default: Default; + } + + export interface Default { + path_word: string; + count: number; + name: string; + } + } + + declare namespace Chapter { + export interface Data { + code: number; + message: string; + results: Results; + } + + export interface Results { + list: List[]; + total: number; + limit: number; + offset: number; + } + + export interface List { + index: number; + uuid: string; + count: number; + ordered: number; + size: number; + name: string; + comic_id: string; + comic_path_word: string; + group_id: any; + group_path_word: string; + type: number; + img_type: number; + news: string; + datetime_created: string; + prev?: string; + next?: string; + } + } + + declare namespace Images { + export interface Data { + code: number; + message: string; + results: Results; + } + + export interface Results { + show_app: boolean; + is_lock: boolean; + is_login: boolean; + is_mobile_bind: boolean; + is_vip: boolean; + comic: Comic; + chapter: Chapter; + } + + export interface Comic { + name: string; + uuid: string; + path_word: string; + restrict: Restrict; + } + + export interface Restrict { + value: number; + display: string; + } + + export interface Chapter { + index: number; + uuid: string; + count: number; + ordered: number; + size: number; + name: string; + comic_id: string; + comic_path_word: string; + group_id: any; + group_path_word: string; + type: number; + img_type: number; + news: string; + datetime_created: string; + prev: string; + next: string; + contents: Content[]; + words: number[]; + is_long: boolean; + } + + export interface Content { + url: string; + } + } +} diff --git a/src/modules/copymanga/index.ts b/src/modules/copymanga/index.ts new file mode 100644 index 0000000..43b2bd2 --- /dev/null +++ b/src/modules/copymanga/index.ts @@ -0,0 +1,109 @@ +/// + +import dayjs from "dayjs"; +import ComicDownloader from "../../comic-downloader"; + +const API_HEADERS = { + "User-Agent": + '"User-Agent" to "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.124 Safari/537.36 Edg/102.0.1245.44"', + version: dayjs().format("YYYY.MM.DD"), + platform: "1", + region: "0", +}; + +const API_URL = "https://api.copymanga.org/"; + +export default class CopymangaDownloader extends ComicDownloader { + static siteName = "copymanga"; + + static canHandleUrl(url: string): boolean { + console.log(url); + return /copymanga/.test(url); + } + + constructor(protected destination: string, protected configs: Configs = {}) { + super(destination, configs); + this.axios.defaults.baseURL = API_URL; + Object.keys(API_HEADERS).forEach((key) => { + this.axios.defaults.headers[key] = + API_HEADERS[key as keyof typeof API_HEADERS] ?? ""; + }); + this.axios.defaults.headers.webp = configs.format === "jpg" ? 0 : 1; + } + + async getSegmentedChapters( + mangaId: string, + page: number + ): Promise { + const res = await this.axios.get( + `/api/v3/comic/${mangaId}/group/default/chapters?limit=500&offset=${ + page * 500 + }&platform=3` + ); + return res?.data?.results?.list?.map((item) => { + const uri = `/api/v3/comic/${mangaId}/chapter/${item.uuid}?platform=3`; + return { + index: item.index, + name: item.name, + uri, + }; + }); + } + + getMangaId(url: string) { + return new URL(url).pathname.split("/").pop(); + } + + async getSerieInfo(url: string): Promise { + const mangaId = this.getMangaId(url); + if (!mangaId) { + throw new Error("Invalid URL."); + } + const res = await this.axios.get( + `/api/v3/comic2/${mangaId}` + ); + const data = res?.data?.results?.comic; + const count = res?.data?.results?.groups?.default?.count ?? 0; + const title = data?.name; + const info: ComicInfo = { + Manga: "YesAndRightToLeft", + Serie: title, + Summary: data?.brief, + Location: data?.region?.display, + Count: count, + Web: url, + Status: data?.status?.value === 0 ? "Ongoing" : "End", + Penciller: data?.author?.map((e) => e.name)?.join(","), + Tags: data?.theme?.map((e) => e.name)?.join(","), + }; + + const chapters: Chapter[] = []; + const pagination = 500; + const pages = Math.ceil((count || 1) / pagination); + for (let page = 0; page < pages; page += pagination) { + const segment = await this.getSegmentedChapters(mangaId, page); + chapters.push(...segment); + } + + return { title, chapters, info }; + } + + protected async getImageList(url: string): Promise<(string | null)[]> { + const res = await this.axios.get(url); + const data = res?.data?.results?.chapter; + /** + * Copymanga now messes up the order of images, + * but in the API data, there's a chapter.words array that contains the correct page order. + */ + const imageUrls = data?.contents.map((item) => item.url); + const imageOrder = data?.words; + const orderedPages: string[] = []; + if (imageOrder?.length) { + imageOrder.forEach((order, index) => { + orderedPages[order] = imageUrls[index]; + }); + return orderedPages; + } + return imageUrls; + } +} diff --git a/src/modules/dmzj/index.ts b/src/modules/dmzj/index.ts deleted file mode 100644 index a801e62..0000000 --- a/src/modules/dmzj/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import ComicDownloader from "../../comic-downloader"; - -export default class DMZJDownloader extends ComicDownloader { - static readonly siteName = "dmzj"; - - getSerieInfo(url: string): Promise { - throw new Error("Method not implemented."); - } - protected getImageList(url: string): Promise<(string | null)[]> { - throw new Error("Method not implemented."); - } -} diff --git a/src/modules/index.ts b/src/modules/index.ts index 38303d9..f69a2fe 100644 --- a/src/modules/index.ts +++ b/src/modules/index.ts @@ -1 +1,2 @@ export { default as ZeroBywDownloader } from "./zerobyw"; +export { default as CopymangaDownloader } from "./copymanga"; diff --git a/src/modules/manhuaren.ts b/src/modules/manhuaren.ts deleted file mode 100644 index 0d0edac..0000000 --- a/src/modules/manhuaren.ts +++ /dev/null @@ -1,42 +0,0 @@ -import MD5 from "crypto-js/md5"; -import { URL } from "url"; -import ComicDownloader from "../comic-downloader"; - -const PrivateKey = "4e0a48e1c0b54041bce9c8f0e036124d"; - -function generateGSNHash(url: string) { - const parsed = new URL(url); - let s = `${PrivateKey}GET`; - parsed.searchParams.forEach((key) => { - if (key !== "gsn") { - s += key; - const value = parsed.searchParams.get(key); - if (value) { - s += encodeURI(value); - } - } - }); - s += PrivateKey; - return MD5(s); -} - -export default class ManhuarenDownloader extends ComicDownloader { - static readonly siteName = "manhuaren"; - static readonly baseUrl = "http://mangaapi.manhuaren.com"; - - constructor(destination: string, configs: Configs) { - super(destination, configs); - this.axios.defaults.headers["X-Yq-Yqci"] = '{"le": "zh"}'; - this.axios.defaults.headers["User-Agent"] = "okhttp/3.11.0"; - this.axios.defaults.headers["Referer"] = "http://www.dm5.com/dm5api/"; - this.axios.defaults.headers["clubReferer"] = - "http://mangaapi.manhuaren.com/"; - } - - getSerieInfo(url: string): Promise { - throw new Error("Method not implemented."); - } - protected getImageList(url: string): Promise<(string | null)[]> { - throw new Error("Method not implemented."); - } -} diff --git a/src/modules/zerobyw.ts b/src/modules/zerobyw/index.ts similarity index 91% rename from src/modules/zerobyw.ts rename to src/modules/zerobyw/index.ts index c941193..976128e 100644 --- a/src/modules/zerobyw.ts +++ b/src/modules/zerobyw/index.ts @@ -1,5 +1,5 @@ import { JSDOM } from "jsdom"; -import ComicDownloader from "../comic-downloader"; +import ComicDownloader from "../../comic-downloader"; const Selectors = { chapters: ".uk-grid-collapse .muludiv a", @@ -13,6 +13,17 @@ const Selectors = { export default class ZeroBywDownloader extends ComicDownloader { static readonly siteName = "zerobyw"; + static canHandleUrl(url: string): boolean { + return /zerobyw/.test(url); + } + + protected detectBaseUrl(url: string): void { + const match = url.match(/^https?:\/\/[^.]+\.[^/]+/); + if (match?.[0]) { + this.axios.defaults.baseURL = match?.[0]; + } + } + async getSerieInfo(url: string): Promise { const res = await this.axios.get(url); if (!res?.data) throw new Error("Request Failed."); @@ -25,7 +36,6 @@ export default class ZeroBywDownloader extends ComicDownloader { document.querySelector("title")?.textContent?.replace(/[ \s]+/g, "") ?? "Untitled"; info.Serie = title; - this.log(`Found ${title}.`); const chapterElements = document.querySelectorAll( Selectors.chapters @@ -40,7 +50,6 @@ export default class ZeroBywDownloader extends ComicDownloader { }); }); info.Count = chapters.length; - this.log(`Chapter Length: ${chapters.length}`); const tagGroup1 = document.querySelectorAll( Selectors.tags1 From 9709cfb8b70d20a23851bb8f7f16a6c36bce5578 Mon Sep 17 00:00:00 2001 From: Yan Date: Fri, 31 Mar 2023 20:21:23 +0200 Subject: [PATCH 3/3] feat(options): override --- CHANGELOG.md | 2 +- README.md | 109 +++++++++++++++++++-------------- package.json | 13 ++-- src/comic-downloader.ts | 32 ++++++++-- src/commands.ts | 5 +- src/global.d.ts | 5 +- src/modules/copymanga/index.ts | 1 - 7 files changed, 107 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaf8e1e..9609eca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ Now this library is designed to be used with multiple manga / comic sites. - [library] The code has been refactored and can now add sites as plugins. - [CLI] The `-b, --batch` flag is now default to 1 when not set. - Downloaded images are now renamed by index (01 ~ ). -- Downloaders now ignores downloaded chapters by default, set `configs.override` to `true` or for CLI use `-O, --override` if you want to override. +- Downloaders now ignores downloaded chapters by default, set `options.override` to `true` or for CLI use `-O, --override` if you want to override. ```typescript // Before diff --git a/README.md b/README.md index 8ba218b..99b9b22 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ This library is not for browsers. ## :green_book: Quick Start +### Stable + You need Node.js (LTS or the current version) to run this project. ```bash @@ -44,53 +46,65 @@ npm i zerobyw-dl npx zerobyw-dl help ``` +### Nightly + +```bash +npm i yinyanfr/comic-dl +# or +git clone https://github.com/yinyanfr/comic-dl.git +cd comic-dl +npm i +npx . help +``` + ## :wrench: Cli ``` - Usage: comic-dl [options] [command] - - Commands: - chapter, c, ch Download images from one chapter. - download, d, dl Download chapters from a manga serie. - help Display help - list, l, ls List all chapters of a manga serie. - version Display version - - Options: - -a, --archive Optional: Output zip or cbz archive grouped by chapters. - -b, --batch Optional: Set the number or images to be downloaded simultaneously, default to 1. - -C, --chapters Optional: Only downloading given list of chapters, example: -C 1,2,4,7 - -c, --cookie Optional (but recommanded): Provide the path to a text file that contains your cookie. - -F, --format Optional: the format of downloaded picture, depending on the modules, example: webp / jpg. - -f, --from Optional: Starting chapter when downloading a serie, default to 0. - -h, --help Output usage information - -i, --info Optional: Generate ComicInfo.xml. - -M, --max-title-length Optional: restrict the length of title as the folder name. - -m, --module Optional: Specify the module (site) name. Will attempt to detect module by url if not set. - -n, --name Optional: Proride the serie title and override the folder name. - -o, --output Optional: The path where downloaded files are saved (default to .), setting this flag when using list will save a ComicInfo.xml to the path. - -r, --retry Optional: Automatically re-download chapters with failed images. - -s, --slience Optional: Silence the console output, including the confirm prompt. - -T, --timeout Optional: Override the default 10s request timeout. - -t, --to Optional: Ending chapter when downloading a serie, defaults to chapter.length - 1. - -u, --url The url to the serie or the chapter. - -v, --verbose Optional: Display detailed error message, overrides silence. - -V, --version Output the version number - -y, --yes Optional: Skipping confirmation prompt when downloading series. - -z, --zip-level Optional: zip level for archive, default to 5. - - Examples: - - Download a serie from its 10th chapter to 20th chapter to the given destination, 10 images at a time, output zip archives with ComicInfo.xml by chapter, retry if a chapter is not properly downloaded. - $ npx comic-dl dl -c cookie.txt -f 10 -t 20 -o ~/Download/manga -a zip -r -i -b 10 -u serie_url - - - Download chapter index 0, 4, 12 from a serie - $ npx comic-dl dl -c cookie.txt -o ~/Download/manga -i -u serie_url -c 0,4,12 - - - List all chapters of the given serie. - $ npx comic-dl ls -u serie_url - - - Download a chapter named Chapter1 to current path. - $ npx comic-dl ch -n Chapter1 -u chapter_url -c cookie.txt +Usage: comic-dl [options] [command] + +Commands: + chapter, c, ch Download images from one chapter. + download, d, dl Download chapters from a manga serie. + help Display help + list, l, ls List all chapters of a manga serie. + version Display version + +Options: + -a, --archive Optional: Output zip or cbz archive grouped by chapters. + -b, --batch Optional: Set the number or images to be downloaded simultaneously, default to 1. + -C, --chapters Optional: Only downloading given list of chapters, example: -C 1,2,4,7 + -c, --cookie Optional (but recommanded): Provide the path to a text file that contains your cookie. + -F, --format Optional: the format of downloaded picture, depending on the modules, example: webp / jpg. + -f, --from Optional: Starting chapter when downloading a serie, default to 0. + -h, --help Output usage information + -i, --info Optional: Generate ComicInfo.xml. + -M, --max-title-length Optional: restrict the length of title as the folder name. + -m, --module Optional: Specify the module (site) name. Will attempt to detect module by url if not set. + -n, --name Optional: Proride the serie title and override the folder name. + -o, --output Optional: The path where downloaded files are saved (default to .), setting this flag when using list will save a ComicInfo.xml to the path. + -O, --override Optional: overrides downloaded chapters. + -r, --retry Optional: Automatically re-download chapters with failed images. + -s, --slience Optional: Silence the console output, including the confirm prompt. + -T, --timeout Optional: Override the default 10s request timeout. + -t, --to Optional: Ending chapter when downloading a serie, defaults to chapter.length - 1. + -u, --url The url to the serie or the chapter. + -v, --verbose Optional: Display detailed error message, overrides silence. + -V, --version Output the version number + -y, --yes Optional: Skipping confirmation prompt when downloading series. + -z, --zip-level Optional: zip level for archive, default to 5. + +Examples: + - Download a serie from its 10th chapter to 20th chapter to the given destination, 10 images at a time, output zip archives with ComicInfo.xml by chapter, retry if a chapter is not properly downloaded. + $ npx comic-dl dl -c cookie.txt -f 10 -t 20 -o ~/Download/manga -a zip -r -i -b 10 -u serie_url + + - Download chapter index 0, 4, 12 from a serie, overriding downloaded files. + $ npx comic-dl dl -c cookie.txt -o ~/Download/manga -i -O -u serie_url -c 0,4,12 + + - List all chapters of the given serie. + $ npx comic-dl ls -u serie_url + + - Download a chapter named Chapter1 to current path. + $ npx comic-dl ch -n Chapter1 -u chapter_url -c cookie.txt ``` ## :book: Library @@ -122,6 +136,9 @@ const configs = { // Restrict the length of title's length, in case your file system has such limitation (Optional: default to undefined) maxTitleLength: 30, // Zip level for archives (Optional: default to 5) + zipLevel: 5, + // Format of downloaded image, (Optional: depending on the modules, normally default to webp or jpg) + format: "webp", }; // Optional const downloader = new ZeroBywDownloader(destination, configs); @@ -160,7 +177,8 @@ const options = { rename: undefined, // Optional: Changing the folder name, default to undefined retry: false, // Optional: Automatically re-download chapters with failed images. info: true, // Optional: Generates ComicInfo.xml, default to **false** - chapters: undefined, // Optional: Automatically re-download chapters with failed images + chapters: undefined, // Optional: Array of chapter indexes to download, will download the entire serie if not provided + override: false, // Optional: Overriding downloaded chapters, default to false onProgress: (progress) => { console.log(progress); }, // Optional: Called when a chapter is downloaded or failed to do so @@ -190,6 +208,7 @@ const options = { index: 0, // chapter index title: "Serie Title", info: ComicInfo, // Optional: Generates ComicInfo.xml, please refer to ComicInfo's Documentations + override: false, // Optional: Overriding downloaded chapters, default to false onProgress: (progress) => {}, // Optional: Called when a chapter is downloaded or failed to do so, the same as in serie options }; // Optional diff --git a/package.json b/package.json index 2952cc2..1753687 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "comic-dl", - "version": "2.0.0-alpha", - "description": "Yet another batch downloader for manga / comic sites", + "version": "2.0.0", + "description": "Yet another batch downloader for manga/comic sites", "main": "dist/index.js", "bin": "dist/cli.js", "types": "src/global.d.ts", @@ -13,7 +13,12 @@ }, "keywords": [ "scraper", - "zerobyw" + "zerobyw", + "copymanga", + "manga", + "comic", + "manga-downloader", + "comic-downloader" ], "repository": { "type": "git", @@ -44,4 +49,4 @@ "mime-types": "^2.1.35", "yesno": "^0.4.0" } -} +} \ No newline at end of file diff --git a/src/comic-downloader.ts b/src/comic-downloader.ts index 61f986c..7554e33 100644 --- a/src/comic-downloader.ts +++ b/src/comic-downloader.ts @@ -216,14 +216,33 @@ export default abstract class ComicDownloader { ? archiver("zip", { zlib: { level: this.configs?.zipLevel ?? 5 } }) : undefined; + const skippedProgress = { + index: options?.index, + name, + status: "skipped" as const, + failures: 0, + }; if (this.configs?.archive) { - const archiveStream = fs.createWriteStream( - path.join( - chapterWritePath, - `${name}.${this.configs?.archive === "cbz" ? "cbz" : "zip"}` - ) + const archiveWritePath = path.join( + chapterWritePath, + `${name}.${this.configs?.archive === "cbz" ? "cbz" : "zip"}` ); - archive?.pipe(archiveStream); + if (!options.override && fs.existsSync(archiveWritePath)) { + this.log( + `Skipped: ${options?.index} - ${name} has already been downloaded` + ); + return skippedProgress; + } else { + const archiveStream = fs.createWriteStream(archiveWritePath); + archive?.pipe(archiveStream); + } + } else { + if (!options.override && fs.existsSync(chapterWritePath)) { + this.log( + `Skipped: ${options?.index} - ${name} has already been downloaded` + ); + return skippedProgress; + } } let failures = 0; @@ -313,6 +332,7 @@ export default abstract class ComicDownloader { index: chapter.index, title: options.rename ?? serie.title, info: options.info ? serie.info : undefined, + override: options.override, onProgress: options?.onProgress, }); summary.push(progress); diff --git a/src/commands.ts b/src/commands.ts index 3020fa9..78a81a4 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -118,6 +118,7 @@ export const downloadCommand: Command = async (name, sub, options = {}) => { retry, chapters, info, + override, } = options; let current: Partial = {}; @@ -132,6 +133,7 @@ export const downloadCommand: Command = async (name, sub, options = {}) => { rename, retry, info, + override, chapters: chapters !== undefined ? `${chapters}`.split(",").map((e) => parseInt(e)) @@ -168,7 +170,7 @@ export const downloadCommand: Command = async (name, sub, options = {}) => { }; export const chapterCommand: Command = async (name, sub, options = {}) => { - const { url, name: chapterName, verbose, output } = options; + const { url, name: chapterName, verbose, output, override } = options; try { if (url) { @@ -179,6 +181,7 @@ export const chapterCommand: Command = async (name, sub, options = {}) => { } await downloader.downloadChapter(chapterName ?? "Untitled", url, { info: serie?.info, + override, }); } else { console.log("Please Provide URL."); diff --git a/src/global.d.ts b/src/global.d.ts index 30fe330..9f7c73e 100644 --- a/src/global.d.ts +++ b/src/global.d.ts @@ -14,7 +14,6 @@ interface Configs { maxTitleLength?: number; zipLevel?: number; format?: string; - override?: boolean; } interface Chapter { @@ -66,7 +65,7 @@ interface DownloadProgress { index?: number; name: string; uri?: string; - status: "completed" | "failed"; + status: "completed" | "failed" | "skipped"; failed?: number; } @@ -78,6 +77,7 @@ interface SerieDownloadOptions { chapters?: number[]; confirm?: boolean; info?: boolean; + override?: boolean; onProgress?: (progress: DownloadProgress) => void; } @@ -91,6 +91,7 @@ interface ChapterDownloadOptions { index?: number; title?: string; info?: ComicInfo; + override?: boolean; onProgress?: (progress: DownloadProgress) => void; } diff --git a/src/modules/copymanga/index.ts b/src/modules/copymanga/index.ts index 43b2bd2..147e09c 100644 --- a/src/modules/copymanga/index.ts +++ b/src/modules/copymanga/index.ts @@ -17,7 +17,6 @@ export default class CopymangaDownloader extends ComicDownloader { static siteName = "copymanga"; static canHandleUrl(url: string): boolean { - console.log(url); return /copymanga/.test(url); }