diff --git a/.env b/.env index 77f18d9..ef0e757 100644 --- a/.env +++ b/.env @@ -1,8 +1,16 @@ +NODE_ENV=production + +CRAWL_FORM="kipalog,daynhauhoc,viblopost" # giaphiep is not supported! + KIPALOG_LAST_PAGE=76 DAYNHAUHOC_LAST_PAGE=1358 -GIAPHIEP_LAST_PAGE=1375 VIBLOQUESTION_LAST_PAGE=64 +VIBLOPOST_LAST_PAGE=1350 +# GIAPHIEP_LAST_PAGE=1375 ## For debug -# DAYNHAUHOC_LAST_PAGE=2 -# KIPALOG_LAST_PAGE=2 \ No newline at end of file +# KIPALOG_LAST_PAGE=76 +# DAYNHAUHOC_LAST_PAGE=1358 +# VIBLOQUESTION_LAST_PAGE=64 +# VIBLOPOST_LAST_PAGE=1350 +# GIAPHIEP_LAST_PAGE=1375 \ No newline at end of file diff --git a/app.js b/app.js index 814108f..f0d5975 100644 --- a/app.js +++ b/app.js @@ -5,12 +5,16 @@ const chalk = require('chalk'); const { kipalogCrawler, kipalogPaginateUrl } = require('./crawlers/kipalogCrawler'); const { daynhauhocCrawler, daynhauhocPaginateUrl } = require('./crawlers/daynhauhocCrawler'); +const { giaphiepCrawler, giaphiepPaginateUrl } = require('./crawlers/giaphiepCrawler'); +const { viblopostCrawler, viblopostPaginateUrl } = require('./crawlers/viblopostCrawler'); -const { kipalogLastPage, daynhauhocLastPage, giaphiepLastPage, vibloquestionLastPage } = require('./config/app.config'); +const { kipalogLastPage, daynhauhocLastPage, giaphiepLastPage, vibloquestionLastPage, viblopostLastPage, crawlFrom } = require('./config/app.config'); -const crawlers = { kipalogCrawler, daynhauhocCrawler }; -const paginateUrls = { kipalogPaginateUrl, daynhauhocPaginateUrl }; -const lastPages = { kipalogLastPage, daynhauhocLastPage, giaphiepLastPage, vibloquestionLastPage }; +const { getCurrentTime, sleep, getRandomInt } = require('./shared'); + +const crawlers = { kipalogCrawler, daynhauhocCrawler, giaphiepCrawler, viblopostCrawler }; +const paginateUrls = { kipalogPaginateUrl, daynhauhocPaginateUrl, giaphiepPaginateUrl, viblopostPaginateUrl }; +const lastPages = { kipalogLastPage, daynhauhocLastPage, giaphiepLastPage, vibloquestionLastPage, viblopostLastPage }; const paginateCrawler = async (pageUrls) => { const browser = await puppeteer.launch({ headless: true }); @@ -20,6 +24,10 @@ const paginateCrawler = async (pageUrls) => { for (let i = 0; i < pageUrls.length; i++) { const page = await browser.newPage(); const { url, type } = pageUrls[i]; + await page.setDefaultNavigationTimeout(0); + let delay = getRandomInt(500, 10_000); + console.log(getCurrentTime() + chalk.yellow('Delay... ') + chalk.white.bgRed(`${delay / 1000}s\t`) + chalk.green(url)); + await sleep(delay); await page.goto(url); let articlesJSON = await page.evaluate(() => { @@ -30,6 +38,8 @@ const paginateCrawler = async (pageUrls) => { if (type === 'daynhauhoc') { articles = articles.topic_list.topics; + } else if (type === 'giaphiep' || type === 'viblopost') { + articles = articles.data; } articleCounter += articles.length; @@ -37,19 +47,19 @@ const paginateCrawler = async (pageUrls) => { return await crawlers[`${type}Crawler`](browser, article); }); - const res = await Promise.all(resPromise); + await Promise.all(resPromise); page.close(); } - console.log(chalk.yellow('Crawled successfully: ') + chalk.white.bgRed(`${articleCounter} articles`)); + console.log(getCurrentTime() + chalk.yellow('Crawled successfully: ') + chalk.white.bgRed(`${articleCounter} articles`)); browser.close(); } const crawler = () => { let pageUrls = []; - const types = ['kipalog', 'daynhauhoc']; + const types = crawlFrom; types.map(type => { const end = lastPages[`${type}LastPage`]; diff --git a/config/app.config.js b/config/app.config.js index 95ccfc2..74ae344 100644 --- a/config/app.config.js +++ b/config/app.config.js @@ -1,9 +1,11 @@ module.exports = { - app :{ + app: { }, kipalogLastPage: process.env.KIPALOG_LAST_PAGE, daynhauhocLastPage: process.env.DAYNHAUHOC_LAST_PAGE, giaphiepLastPage: process.env.GIAPHIEP_LAST_PAGE, vibloquestionLastPage: process.env.VIBLOQUESTION_LAST_PAGE, + viblopostLastPage: process.env.VIBLOPOST_LAST_PAGE, + crawlFrom: String(process.env.CRAWL_FROM).split(',') } \ No newline at end of file diff --git a/config/config.json b/config/config.json index 4965cf3..16a7309 100644 --- a/config/config.json +++ b/config/config.json @@ -9,6 +9,7 @@ }, "production": { "storage": "development.db", - "dialect": "sqlite" + "dialect": "sqlite", + "logging": false } } \ No newline at end of file diff --git a/crawlers/daynhauhocCrawler.js b/crawlers/daynhauhocCrawler.js index 8fb6b86..b8772cb 100644 --- a/crawlers/daynhauhocCrawler.js +++ b/crawlers/daynhauhocCrawler.js @@ -2,6 +2,8 @@ const chalk = require('chalk'); const { articleModel } = require('../models'); +const { getCurrentTime, sleep, getRandomInt } = require('../shared'); + const daynhauhocHomePage = 'https://daynhauhoc.com'; const type = 'daynhauhoc'; @@ -20,12 +22,16 @@ const daynhauhocCrawler = async (browser, article) => { let path = `/t/${slug}/${id}`; const pageUrl = `${daynhauhocHomePage}${path}`; - console.log(chalk.yellow('Crawling... ') + chalk.green(pageUrl)); const page = await browser.newPage(); await page.setDefaultNavigationTimeout(0); + let delay = getRandomInt(500, 10_000); + console.log(getCurrentTime() + chalk.yellow('Delay... ') + chalk.white.bgRed(`${delay / 1000}s\t`) + chalk.green(pageUrl)); + await sleep(delay); await page.goto(pageUrl); + console.log(getCurrentTime() + chalk.yellow('Crawling...\t') + chalk.green(pageUrl)); + const { htmlContent, textContent } = await page.evaluate(() => { const raw = document.querySelectorAll('#post_1 > div > div.topic-body.clearfix > div.regular.contents')[0]; const htmlContent = raw.outerHTML; @@ -36,6 +42,7 @@ const daynhauhocCrawler = async (browser, article) => { page.close(); const articleData = { title, path, tags, htmlContent, textContent, from: `${type}` }; await articleModel.create(articleData); + console.log(getCurrentTime() + chalk.yellow('Done:\t\t') + chalk.green(pageUrl)); return articleData; } diff --git a/crawlers/giaphiepCrawler.js b/crawlers/giaphiepCrawler.js index 242cba8..398f0e7 100644 --- a/crawlers/giaphiepCrawler.js +++ b/crawlers/giaphiepCrawler.js @@ -1,2 +1,101 @@ +const chalk = require('chalk'); + +const { articleModel } = require('../models'); +const { sleep, getRandomInt, getCurrentTime } = require('../shared'); + +const giaphiepHomePage = 'https://giaphiep.com'; + +const type = 'giaphiep'; + +const giaphiepPaginateUrl = (page = 0) => { + return `https://api.giaphiep.com/posts?page=${page}`; +} + +const giaphiepCrawler = async (browser, article) => { + let { title, link, tags } = article; + + if (tags.length !== 0) + tags = tags.map(tag => String(tag.name).toLocaleLowerCase().trim().replace(' ', '-')).join(';'); + else + tags = 'none'; + + let path = `/blog/${link}`; + const pageUrl = `${giaphiepHomePage}${path}`; + + const page = await browser.newPage(); + await page.setDefaultNavigationTimeout(0); + + await sleep(getRandomInt(5000, 30_000)); + + await page.goto(pageUrl); + + // retry: while (true) { + // try { + // console.log(getCurrentTime() + chalk.yellow(' Crawling... ') + chalk.green(pageUrl)); + // const { htmlContent, textContent, realPath } = await page.evaluate(() => { + // const raw = document.querySelectorAll('#__layout > div > div.el-col.el-col-20 > div:nth-child(1) > div.el-col.el-col-24.el-col-md-18 > div:nth-child(2) > div > div > div.content')[0]; + // const realPathRaw = document.querySelectorAll('#__layout > div > div.el-col.el-col-20 > div:nth-child(1) > div.el-col.el-col-24.el-col-md-18 > div:nth-child(2) > div > div > div:nth-child(6) > h3 > a')[0]; + // const htmlContent = raw.outerHTML; + // const { textContent } = raw; + // const realPath = realPathRaw.textContent.replace('https://viblo.asia', '');; + // return { htmlContent, textContent, realPath }; + // }); + + // page.close(); + // const articleData = { title, path: realPath, tags, htmlContent, textContent, from: `viblo` }; + // await articleModel.create(articleData); + // return articleData; + // } catch (e) { + // const delay = getRandomInt(5000, 100_000); + // console.log(getCurrentTime() + chalk.yellow('Re-try crawling... ' + chalk.green(pageUrl)) + chalk.white.bgRed(` after ${delay / 1000}s`)); + // await sleep(delay); + // await page.reload({ waitUntil: ["networkidle0", "domcontentloaded"] }); + // console.log(getCurrentTime() + chalk.yellow('Crawling... ' + chalk.green(pageUrl))); + // continue retry; + // } + // } + + console.log(getCurrentTime() + chalk.yellow(' Crawling... ') + chalk.green(pageUrl)); + const { htmlContent, textContent, realPath } = await page.evaluate(async () => { + const sleep = (ms = 1000) => { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + const getRandomInt = (min, max) => { + min = Math.ceil(min); + max = Math.floor(max); + return Math.floor(Math.random() * (max - min + 1)) + min; + } + + let raw = null; + retry: while (true) { + try { + raw = document.querySelectorAll('#__layout > div > div.el-col.el-col-20 > div:nth-child(1) > div.el-col.el-col-24.el-col-md-18 > div:nth-child(2) > div > div > div.content')[0]; + const realPathRaw = document.querySelectorAll('#__layout > div > div.el-col.el-col-20 > div:nth-child(1) > div.el-col.el-col-24.el-col-md-18 > div:nth-child(2) > div > div > div:nth-child(6) > h3 > a')[0]; + const htmlContent = raw.outerHTML; + const { textContent } = raw; + const realPath = realPathRaw.textContent.replace('https://viblo.asia', ''); + return { htmlContent, textContent, realPath }; + } catch (e) { + await sleep(getRandomInt(5000, 30_000)); + await location.reload(); + continue retry; + } + } + }); + + page.close(); + const articleData = { title, path: realPath, tags, htmlContent, textContent, from: `viblo` }; + await articleModel.create(articleData); + return articleData; +} + +module.exports = { + giaphiepPaginateUrl, + giaphiepCrawler, + giaphiepHomePage, + type +} + // max=1375x20 // https://api.giaphiep.com/posts?page=1375 \ No newline at end of file diff --git a/crawlers/kipalogCrawler.js b/crawlers/kipalogCrawler.js index 0a5247f..49a3ea0 100644 --- a/crawlers/kipalogCrawler.js +++ b/crawlers/kipalogCrawler.js @@ -2,6 +2,8 @@ const chalk = require('chalk'); const { articleModel } = require('../models'); +const { getCurrentTime, sleep, getRandomInt } = require('../shared'); + const kipalogHomePage = 'https://kipalog.com'; const type = 'kipalog'; @@ -19,12 +21,16 @@ const kipalogCrawler = async (browser, article) => { tags = 'none'; const pageUrl = `${kipalogHomePage}${path}`; - console.log(chalk.yellow('Crawling... ') + chalk.green(pageUrl)); const page = await browser.newPage(); await page.setDefaultNavigationTimeout(0); + let delay = getRandomInt(500, 10_000); + console.log(getCurrentTime() + chalk.yellow('Delay... ') + chalk.white.bgRed(`${delay / 1000}s\t`) + chalk.green(pageUrl)); + await sleep(delay); await page.goto(pageUrl); + console.log(getCurrentTime() + chalk.yellow('Crawling...\t') + chalk.green(pageUrl)); + const { htmlContent, textContent } = await page.evaluate(() => { const raw = document.getElementById('content'); const htmlContent = raw.outerHTML; @@ -35,6 +41,7 @@ const kipalogCrawler = async (browser, article) => { page.close(); const articleData = { title, path, tags, htmlContent, textContent, from: `${type}` }; await articleModel.create(articleData); + console.log(getCurrentTime() + chalk.yellow('Done:\t\t') + chalk.green(pageUrl)); return articleData; } diff --git a/crawlers/viblopostCrawler.js b/crawlers/viblopostCrawler.js new file mode 100644 index 0000000..9d9dd01 --- /dev/null +++ b/crawlers/viblopostCrawler.js @@ -0,0 +1,48 @@ +const chalk = require('chalk'); +const showdown = require('showdown'); + +const converter = new showdown.Converter(); + +const { articleModel } = require('../models'); + +const { getCurrentTime } = require('../shared'); + +const viblopostHomePage = 'https://viblo.asia'; + +const type = 'viblopost'; + +const viblopostPaginateUrl = (page = 0) => { + return `${viblopostHomePage}/api/posts/newest?page=${page}&limit=20`; +} + +const viblopostCrawler = async (browser, article) => { + let { title, url, tags, contents } = article; + + console.log(getCurrentTime() + chalk.yellow('Crawling...\t') + chalk.green(url)); + + tags = tags.data; + + if (tags.length !== 0) + tags = tags.map(tag => String(tag.name).toLocaleLowerCase().trim().replace(' ', '-')).join(';'); + else + tags = 'none'; + + let path = url.replace(viblopostHomePage, ''); + let textContent = contents; + let htmlContent = converter.makeHtml(contents); + + const articleData = { title, path, tags, htmlContent, textContent, from: `${type}` }; + await articleModel.create(articleData); + console.log(getCurrentTime() + chalk.yellow('Done:\t\t') + chalk.green(url)); + return articleData; +} + +module.exports = { + viblopostPaginateUrl, + viblopostCrawler, + viblopostHomePage, + type +} + +// max = 1350 +// https://viblo.asia/api/posts/newest?page=4&limit=20 \ No newline at end of file diff --git a/package.json b/package.json index 080a5a1..58c6c32 100644 --- a/package.json +++ b/package.json @@ -4,13 +4,17 @@ "main": "index.js", "license": "MIT", "scripts": { - "start": "node app.js" + "start": "node app.js", + "db:migrate": "sequelize db:migrate", + "db:new": "rm -f development.db && sequelize db:migrate" }, "dependencies": { "chalk": "^4.1.0", "dotenv": "^8.2.0", + "moment": "^2.27.0", "puppeteer": "^5.2.1", "sequelize": "^6.3.5", + "showdown": "^1.9.1", "sqlite3": "^5.0.0" } } diff --git a/readme.md b/readme.md index 75aad7f..40676bc 100644 --- a/readme.md +++ b/readme.md @@ -22,7 +22,13 @@ yarn install ## Migration ```bash -sequelize db:migrate +yarn db:migrate +``` + +OR - remove old DB and re-migrate + +```bash +yarn db:new # this command will remove developement.db file & migrate DB ! ``` ## Start crawling data diff --git a/shared/index.js b/shared/index.js new file mode 100644 index 0000000..254ab63 --- /dev/null +++ b/shared/index.js @@ -0,0 +1,21 @@ +const moment = require('moment'); + +function getCurrentTime() { + let time = `[${moment().format('DD/MM/YYYY, h:m:s A')}]`; + time += time.length < 24 ? '\t\t' : '\t'; + return time; +} + +function sleep(ms = 1000) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function getRandomInt(min, max) { + min = Math.ceil(min); + max = Math.floor(max); + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +module.exports = { + sleep, getRandomInt, getCurrentTime +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 3490e4c..14ad78d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -49,6 +49,18 @@ ansi-regex@^3.0.0: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" integrity sha1-7QMXwyIGT3lGbAKWa922Bas32Zg= +ansi-regex@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997" + integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== + +ansi-styles@^3.2.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== + dependencies: + color-convert "^1.9.0" + ansi-styles@^4.1.0: version "4.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.2.1.tgz#90ae75c424d008d2624c5bf29ead3177ebfcf359" @@ -156,6 +168,11 @@ buffer@^5.2.1, buffer@^5.5.0: base64-js "^1.0.2" ieee754 "^1.1.4" +camelcase@^5.0.0: + version "5.3.1" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" + integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== + caseless@~0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" @@ -174,11 +191,27 @@ chownr@^1.1.1: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== +cliui@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-5.0.0.tgz#deefcfdb2e800784aa34f46fa08e06851c7bbbc5" + integrity sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA== + dependencies: + string-width "^3.1.0" + strip-ansi "^5.2.0" + wrap-ansi "^5.1.0" + code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + color-convert@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" @@ -186,6 +219,11 @@ color-convert@^2.0.1: dependencies: color-name "~1.1.4" +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= + color-name@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" @@ -234,6 +272,11 @@ debug@^3.2.6: dependencies: ms "^2.1.1" +decamelize@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" + integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= + deep-extend@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" @@ -277,6 +320,11 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" +emoji-regex@^7.0.1: + version "7.0.3" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156" + integrity sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA== + end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" @@ -327,6 +375,13 @@ fd-slicer@~1.1.0: dependencies: pend "~1.2.0" +find-up@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73" + integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg== + dependencies: + locate-path "^3.0.0" + find-up@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" @@ -390,6 +445,11 @@ gauge@~2.7.3: strip-ansi "^3.0.1" wide-align "^1.1.0" +get-caller-file@^2.0.1: + version "2.0.5" + resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" + integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== + get-stream@^5.1.0: version "5.2.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" @@ -565,6 +625,14 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.10.0" +locate-path@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e" + integrity sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A== + dependencies: + p-locate "^3.0.0" + path-exists "^3.0.0" + locate-path@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0" @@ -640,7 +708,7 @@ moment-timezone@^0.5.31: dependencies: moment ">= 2.9.0" -"moment@>= 2.9.0", moment@^2.26.0: +"moment@>= 2.9.0", moment@^2.26.0, moment@^2.27.0: version "2.27.0" resolved "https://registry.yarnpkg.com/moment/-/moment-2.27.0.tgz#8bff4e3e26a236220dfe3e36de756b6ebaa0105d" integrity sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ== @@ -784,13 +852,20 @@ osenv@0, osenv@^0.1.4: os-homedir "^1.0.0" os-tmpdir "^1.0.0" -p-limit@^2.2.0: +p-limit@^2.0.0, p-limit@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== dependencies: p-try "^2.0.0" +p-locate@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4" + integrity sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ== + dependencies: + p-limit "^2.0.0" + p-locate@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07" @@ -803,6 +878,11 @@ p-try@^2.0.0: resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== +path-exists@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" + integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU= + path-exists@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" @@ -944,6 +1024,16 @@ request@^2.87.0: tunnel-agent "^0.6.0" uuid "^3.3.2" +require-directory@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" + integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= + +require-main-filename@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" + integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== + retry-as-promised@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/retry-as-promised/-/retry-as-promised-3.2.0.tgz#769f63d536bec4783549db0777cb56dadd9d8543" @@ -1024,11 +1114,18 @@ sequelize@^6.3.5: validator "^10.11.0" wkx "^0.5.0" -set-blocking@~2.0.0: +set-blocking@^2.0.0, set-blocking@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= +showdown@^1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/showdown/-/showdown-1.9.1.tgz#134e148e75cd4623e09c21b0511977d79b5ad0ef" + integrity sha512-9cGuS382HcvExtf5AHk7Cb4pAeQQ+h0eTr33V1mu+crYWV4KvWAw6el92bDrqGEk5d46Ai/fhbEUwqJ/mTCNEA== + dependencies: + yargs "^14.2" + signal-exit@^3.0.0: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" @@ -1076,6 +1173,15 @@ string-width@^1.0.1: is-fullwidth-code-point "^2.0.0" strip-ansi "^4.0.0" +string-width@^3.0.0, string-width@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961" + integrity sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w== + dependencies: + emoji-regex "^7.0.1" + is-fullwidth-code-point "^2.0.0" + strip-ansi "^5.1.0" + string_decoder@^1.1.1: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" @@ -1104,6 +1210,13 @@ strip-ansi@^4.0.0: dependencies: ansi-regex "^3.0.0" +strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae" + integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA== + dependencies: + ansi-regex "^4.1.0" + strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" @@ -1233,6 +1346,11 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" +which-module@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" + integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho= + which@1: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" @@ -1254,6 +1372,15 @@ wkx@^0.5.0: dependencies: "@types/node" "*" +wrap-ansi@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-5.1.0.tgz#1fd1f67235d5b6d0fee781056001bfb694c03b09" + integrity sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q== + dependencies: + ansi-styles "^3.2.0" + string-width "^3.0.0" + strip-ansi "^5.0.0" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" @@ -1264,11 +1391,41 @@ ws@^7.2.3: resolved "https://registry.yarnpkg.com/ws/-/ws-7.3.1.tgz#d0547bf67f7ce4f12a72dfe31262c68d7dc551c8" integrity sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA== +y18n@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b" + integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w== + yallist@^3.0.0, yallist@^3.0.3: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== +yargs-parser@^15.0.1: + version "15.0.1" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-15.0.1.tgz#54786af40b820dcb2fb8025b11b4d659d76323b3" + integrity sha512-0OAMV2mAZQrs3FkNpDQcBk1x5HXb8X4twADss4S0Iuk+2dGnLOE/fRHrsYm542GduMveyA77OF4wrNJuanRCWw== + dependencies: + camelcase "^5.0.0" + decamelize "^1.2.0" + +yargs@^14.2: + version "14.2.3" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-14.2.3.tgz#1a1c3edced1afb2a2fea33604bc6d1d8d688a414" + integrity sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg== + dependencies: + cliui "^5.0.0" + decamelize "^1.2.0" + find-up "^3.0.0" + get-caller-file "^2.0.1" + require-directory "^2.1.1" + require-main-filename "^2.0.0" + set-blocking "^2.0.0" + string-width "^3.0.0" + which-module "^2.0.0" + y18n "^4.0.0" + yargs-parser "^15.0.1" + yauzl@^2.10.0: version "2.10.0" resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"