From 376ebf4bc46db3e4d42bf4997fb60e7e07fc93c5 Mon Sep 17 00:00:00 2001 From: Ethan Shen <42264778+nczitzk@users.noreply.github.com> Date: Sun, 14 Jan 2024 16:02:07 +0800 Subject: [PATCH 1/6] feat(route): add TradingView Desktop releases and release notes (#14234) --- lib/v2/tradingview/desktop.js | 61 ++++++++++++++++++++++++++ lib/v2/tradingview/maintainer.js | 1 + lib/v2/tradingview/radar.js | 6 +++ lib/v2/tradingview/router.js | 1 + website/docs/routes/program-update.mdx | 4 ++ 5 files changed, 73 insertions(+) create mode 100644 lib/v2/tradingview/desktop.js diff --git a/lib/v2/tradingview/desktop.js b/lib/v2/tradingview/desktop.js new file mode 100644 index 00000000000000..e894fae40428a4 --- /dev/null +++ b/lib/v2/tradingview/desktop.js @@ -0,0 +1,61 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const timezone = require('@/utils/timezone'); +const { parseDate } = require('@/utils/parse-date'); + +module.exports = async (ctx) => { + const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 50; + + const rootUrl = 'https://www.tradingview.com'; + const currentUrl = new URL('/support/solutions/43000673888-tradingview-desktop-releases-and-release-notes/', rootUrl).href; + + const { data: response } = await got(currentUrl); + + const $ = cheerio.load(response); + + $('h4[data-identifyelement]').each((_, el) => { + el = $(el); + + if (el.text().trim() === '') { + el.remove(); + } + }); + + const items = $('h4[data-identifyelement]') + .toArray() + .slice(0, limit) + .map((item) => { + item = $(item); + + const title = item.text(); + const description = $.html(item.nextUntil('h4')); + const content = cheerio.load(description); + + return { + title, + link: currentUrl, + description, + category: content('h5') + .toArray() + .map((c) => $(c).text()), + guid: `tradingview-desktop#${title.split(/versions?\s/).pop()}`, + pubDate: timezone(parseDate(title.split(/\./)[0], 'MMMM D, YYYY'), +8), + }; + }); + + const title = $('title').text(); + const titleSplits = title.split(/—/); + const icon = new URL($('link[rel="icon"]').prop('href'), rootUrl).href; + + ctx.state.data = { + item: items, + title, + link: currentUrl, + description: titleSplits[0], + language: $('html').prop('lang'), + icon, + logo: icon, + subtitle: titleSplits[0], + author: titleSplits.pop(), + }; +}; diff --git a/lib/v2/tradingview/maintainer.js b/lib/v2/tradingview/maintainer.js index cb9d37b3a3a949..10e854fa3281b6 100644 --- a/lib/v2/tradingview/maintainer.js +++ b/lib/v2/tradingview/maintainer.js @@ -1,3 +1,4 @@ module.exports = { '/blog/:language?/category/:category?': ['nczitzk'], + '/desktop': ['nczitzk'], }; diff --git a/lib/v2/tradingview/radar.js b/lib/v2/tradingview/radar.js index 642caa3f60af72..04884af51e58a4 100644 --- a/lib/v2/tradingview/radar.js +++ b/lib/v2/tradingview/radar.js @@ -92,6 +92,12 @@ module.exports = { source: ['/blog/:language/category/widgets/'], target: '/tradingview/blog/:language/category/widgets', }, + { + title: 'Desktop releases and release notes', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-desktop-releases-and-release-notes', + source: ['/support/solutions/43000673888-tradingview-desktop-releases-and-release-notes/'], + target: '/tradingview/desktop', + }, ], }, }; diff --git a/lib/v2/tradingview/router.js b/lib/v2/tradingview/router.js index 98bb22bf2c0d0a..a985e2dcc14ac0 100644 --- a/lib/v2/tradingview/router.js +++ b/lib/v2/tradingview/router.js @@ -1,3 +1,4 @@ module.exports = function (router) { router.get('/blog/:category*', require('./blog')); + router.get('/desktop', require('./desktop')); }; diff --git a/website/docs/routes/program-update.mdx b/website/docs/routes/program-update.mdx index ccf012212ff8e0..a03328077f5e5f 100644 --- a/website/docs/routes/program-update.mdx +++ b/website/docs/routes/program-update.mdx @@ -616,6 +616,10 @@ Logseq 开发团队已经放弃了 [旧网站](https://logseq.com/blog)。 | [Widgets](https://www.tradingview.com/blog/en/category/widgets/) | category/widgets | +### Desktop releases and release notes {#tradingview-desktop-releases-and-release-notes} + + + ## Typora {#typora} ### Changelog {#typora-changelog} From 418e68aa57b36a5c6e6db4816fcd204c555a9a38 Mon Sep 17 00:00:00 2001 From: huanfei <41602338+huanfe1@users.noreply.github.com> Date: Sun, 14 Jan 2024 17:35:00 +0800 Subject: [PATCH 2/6] fix: pornhub pubDate (#14232) --- lib/v2/pornhub/utils.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/v2/pornhub/utils.js b/lib/v2/pornhub/utils.js index 10d4af2b4663c3..1dc21ff9a540d6 100644 --- a/lib/v2/pornhub/utils.js +++ b/lib/v2/pornhub/utils.js @@ -1,6 +1,7 @@ const { art } = require('@/utils/render'); const { join } = require('path'); const { parseRelativeDate } = require('@/utils/parse-date'); +const dayjs = require('dayjs'); const defaultDomain = 'https://www.pornhub.com'; @@ -10,6 +11,10 @@ const headers = { }; const renderDescription = (data) => art(join(__dirname, 'templates/description.art'), data); +const extractDateFromImageUrl = (imageUrl) => { + const matchResult = imageUrl.match(/(\d{6})\/(\d{2})/); + return matchResult ? matchResult.slice(1, 3).join('') : null; +}; const parseItems = (e) => ({ title: e.find('span.title a').text().trim(), @@ -19,7 +24,7 @@ const parseItems = (e) => ({ previewVideo: e.find('img').data('mediabook'), }), author: e.find('.usernameWrap a').text(), - pubDate: parseRelativeDate(e.find('.added').text()), + pubDate: dayjs(extractDateFromImageUrl(e.find('img').data('mediumthumb'))) || parseRelativeDate(e.find('.added').text()), }); module.exports = { From 1a49f937d8c4aea2bc9cc1dc2cbc9614270043fd Mon Sep 17 00:00:00 2001 From: JimenezLi <75196426+JimenezLi@users.noreply.github.com> Date: Sun, 14 Jan 2024 23:21:19 +0800 Subject: [PATCH 3/6] fix(route): fix twitch (#14238) * fix(route): fix twitch * fix docs --- lib/v2/twitch/schedule.js | 3 ++- lib/v2/twitch/video.js | 21 ++++++++++++++------- website/docs/routes/live.mdx | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/lib/v2/twitch/schedule.js b/lib/v2/twitch/schedule.js index 21345e196f5409..b393fc8fdeddc4 100644 --- a/lib/v2/twitch/schedule.js +++ b/lib/v2/twitch/schedule.js @@ -57,7 +57,8 @@ module.exports = async (ctx) => { const displayName = channelShellData.userOrError.displayName; - const out = streamScheduleData.user.channel.schedule.segments.map((item) => ({ + // schedule segments may be null + const out = streamScheduleData.user.channel.schedule.segments?.map((item) => ({ title: item.title, guid: item.id, link: `https://www.twitch.tv/${login}`, diff --git a/lib/v2/twitch/video.js b/lib/v2/twitch/video.js index d552077d98c815..a29c5443fefdc4 100644 --- a/lib/v2/twitch/video.js +++ b/lib/v2/twitch/video.js @@ -4,14 +4,18 @@ const { parseDate } = require('@/utils/parse-date'); // https://github.com/streamlink/streamlink/blob/master/src/streamlink/plugins/twitch.py#L286 const TWITCH_CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'; -const FILTER_CURSOR_MAP = { - archive: 0, - highlights: 1, - all: 2, +const FILTER_NODE_TYPE_MAP = { + archive: 'LATEST_BROADCASTS', + highlights: 'LATEST_NON_BROADCASTS', + all: 'ALL_VIDEOS', }; module.exports = async (ctx) => { - const { login, filter = 'all' } = ctx.params; + const login = ctx.params.login; + const filter = ctx.params.filter?.toLowerCase() || 'all'; + if (!FILTER_NODE_TYPE_MAP[filter]) { + throw Error(`Unsupported filter type "${filter}", please choose from { ${Object.keys(FILTER_NODE_TYPE_MAP).join(', ')} }`); + } const response = await got({ method: 'post', @@ -45,7 +49,10 @@ module.exports = async (ctx) => { const displayName = channelVideoShelvesQueryData.user.displayName; - const videoShelvesEdge = channelVideoShelvesQueryData.user.videoShelves.edges[FILTER_CURSOR_MAP[filter] || FILTER_CURSOR_MAP.all]; + const videoShelvesEdge = channelVideoShelvesQueryData.user.videoShelves.edges.find((edge) => edge.node.type === FILTER_NODE_TYPE_MAP[filter]); + if (!videoShelvesEdge) { + throw Error(`No video under filter type "${filter}"`); + } const out = videoShelvesEdge.node.items.map((item) => ({ title: item.title, @@ -53,7 +60,7 @@ module.exports = async (ctx) => { author: displayName, pubDate: parseDate(item.publishedAt), description: `
`, - category: [item.game.displayName], + category: item.game && [item.game.displayName], // item.game may be null })); ctx.state.data = { diff --git a/website/docs/routes/live.mdx b/website/docs/routes/live.mdx index 1e92b84d4f6ef7..7cf6de50cc588d 100644 --- a/website/docs/routes/live.mdx +++ b/website/docs/routes/live.mdx @@ -18,7 +18,7 @@ ### Channel Video {#twitch-channel-video} - + | archive | highlights | all | | ----------------- | ----------------------------- | ---------- | From 7a94c42e943ecc56531691fb186ecdf33a25eeb0 Mon Sep 17 00:00:00 2001 From: Ethan Shen <42264778+nczitzk@users.noreply.github.com> Date: Mon, 15 Jan 2024 01:25:20 +0800 Subject: [PATCH 4/6] =?UTF-8?q?feat(route):=20add=20=E8=8B=8F=E5=B7=9E?= =?UTF-8?q?=E5=B8=82=E5=8F=91=E5=B1=95=E5=92=8C=E6=94=B9=E9=9D=A9=E5=A7=94?= =?UTF-8?q?=E5=91=98=E4=BC=9A=20(#14214)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(route): add 苏州市发展和改革委员会 * docs: remove duplicated heading * refactor: migrate to v2 * fix: suzhou docs * fix: news --------- --- lib/router.js | 4 +- lib/routes/gov/suzhou/doc.js | 39 -------- lib/routes/gov/suzhou/news.js | 142 ----------------------------- lib/routes/gov/suzhou/utils.js | 20 ---- lib/v2/gov/maintainer.js | 3 + lib/v2/gov/radar.js | 25 +++++ lib/v2/gov/router.js | 3 + lib/v2/gov/suzhou/doc.js | 46 ++++++++++ lib/v2/gov/suzhou/fg.js | 62 +++++++++++++ lib/v2/gov/suzhou/news.js | 116 +++++++++++++++++++++++ website/docs/routes/government.mdx | 12 ++- 11 files changed, 267 insertions(+), 205 deletions(-) delete mode 100644 lib/routes/gov/suzhou/doc.js delete mode 100644 lib/routes/gov/suzhou/news.js delete mode 100644 lib/routes/gov/suzhou/utils.js create mode 100644 lib/v2/gov/suzhou/doc.js create mode 100644 lib/v2/gov/suzhou/fg.js create mode 100644 lib/v2/gov/suzhou/news.js diff --git a/lib/router.js b/lib/router.js index d68905006b0ae0..09b80aaeb2a61d 100644 --- a/lib/router.js +++ b/lib/router.js @@ -487,8 +487,8 @@ router.get('/gov/shuju/:caty/:item', lazyloadRouteHandler('./routes/gov/shuju')) router.get('/gov/xinwen/tujie/:caty', lazyloadRouteHandler('./routes/gov/xinwen/tujie')); // 苏州 -router.get('/gov/suzhou/news/:uid', lazyloadRouteHandler('./routes/gov/suzhou/news')); -router.get('/gov/suzhou/doc', lazyloadRouteHandler('./routes/gov/suzhou/doc')); +// router.get('/gov/suzhou/news/:uid', lazyloadRouteHandler('./routes/gov/suzhou/news')); +// router.get('/gov/suzhou/doc', lazyloadRouteHandler('./routes/gov/suzhou/doc')); // 山西 router.get('/gov/shanxi/rst/:category', lazyloadRouteHandler('./routes/gov/shanxi/rst')); diff --git a/lib/routes/gov/suzhou/doc.js b/lib/routes/gov/suzhou/doc.js deleted file mode 100644 index 4baa1bc3d13392..00000000000000 --- a/lib/routes/gov/suzhou/doc.js +++ /dev/null @@ -1,39 +0,0 @@ -const got = require('@/utils/got'); -const cheerio = require('cheerio'); -const util = require('./utils'); -module.exports = async (ctx) => { - const link = 'https://www.suzhou.gov.cn/szxxgk/front/xxgk_right.jsp'; - const response = await got.get(link); - const data = response.data; - const $ = cheerio.load(data); - const list = $('.tr_main_value_odd'); - - ctx.state.data = { - title: '苏州市政府 - 政策公开文件', - link, - item: await Promise.all( - list - .slice(0, 12) - .map(async (index, item) => { - item = $(item); - // 获取全文 - const contentUrl = 'https://www.suzhou.gov.cn' + item.find('a').attr('href'); - const arr = await ctx.cache.tryGet(contentUrl, async () => { - const fullText = await got.get(contentUrl); - const fullTextData = cheerio.load(fullText.data); - const content = util.content(fullText.data); - const title = fullTextData('h1').text().replace(/\s*/g, ''); - - return new Array(title, content); - }); - return { - title: arr[0], - description: arr[1], - pubDate: item.find('td:nth-child(4)').text(), - link: contentUrl, - }; - }) - .get() - ), - }; -}; diff --git a/lib/routes/gov/suzhou/news.js b/lib/routes/gov/suzhou/news.js deleted file mode 100644 index ee5e278e6b3685..00000000000000 --- a/lib/routes/gov/suzhou/news.js +++ /dev/null @@ -1,142 +0,0 @@ -const got = require('@/utils/got'); -const cheerio = require('cheerio'); -const logger = require('@/utils/logger'); -const liburl = require('url'); -const util = require('./utils'); -const root_url = 'https://www.suzhou.gov.cn/'; - -module.exports = async (ctx) => { - const uid = ctx.params.uid; - let url = ''; - let title = ''; - let urljs = ''; - switch (uid) { - case 'szyw': - case 'news': - urljs = 'https://www.suzhou.gov.cn/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=5057aeffb1a84a7e8aeded87728da48c'; - url = 'https://www.suzhou.gov.cn/szsrmzf/szyw/nav_list.shtml'; - title = '苏州市政府 - 苏州要闻'; - break; - case 'qxkx': - case 'district': - urljs = 'https://www.suzhou.gov.cn/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=75c636ea0efb487ea7e479e3cc0ff3e5'; - url = 'https://www.suzhou.gov.cn/szsrmzf/qxkx/nav_list.shtml'; - title = '苏州市政府 - 区县快讯'; - break; - case 'bmdt': - urljs = 'https://www.suzhou.gov.cn/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=b3d097e3eb79421f88439ea381ce33c3'; - url = 'https://www.suzhou.gov.cn/szsrmzf/bmdt/nav_list.shtml'; - title = '苏州市政府 - 部门动态'; - break; - case 'xwsp': - urljs = 'https://www.suzhou.gov.cn/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=507980d214c943ebb0a70853ec94b12e'; - url = 'https://www.suzhou.gov.cn/szsrmzf/xwsp/nav_list.shtml'; - title = '苏州市政府 - 新闻视频'; - break; - case 'rdzt': - url = 'https://www.suzhou.gov.cn/szsrmzf/rdzt/nav_list.shtml'; - title = '苏州市政府 - 热点专题'; - break; - case 'sbjzt': - url = 'https://www.suzhou.gov.cn/szsrmzf/sbjzt/nav_list.shtml'; - title = '苏州市政府 - 市本级专题'; - break; - case 'zxrdzt': - url = 'https://www.suzhou.gov.cn/szsrmzf/zxrdzt/nav_list.shtml'; - title = '苏州市政府 - 最新热点专题'; - break; - case 'wqzt': - url = 'https://www.suzhou.gov.cn/szsrmzf/wqzt/nav_list.shtml'; - title = '苏州市政府 - 往期专题'; - break; - case 'qxzt': - url = 'https://www.suzhou.gov.cn/szsrmzf/qxzt/nav_list.shtml'; - title = '苏州市政府 - 区县专题'; - break; - case 'zwgg': - urljs = 'https://www.suzhou.gov.cn/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=260915178a1f4c4fac44c4bf6378c9b0'; - url = 'https://www.suzhou.gov.cn/szsrmzf/zwgg/nav_list.shtml'; - title = '苏州市政府 - 政务公告'; - break; - case 'mszx': - urljs = 'https://www.suzhou.gov.cn/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=dc60acecb0be46b89d42272dcb8bd32b'; - url = 'https://www.suzhou.gov.cn/szsrmzf/mszx/nav_list.shtml'; - title = '苏州市政府 - 便民公告'; - break; - case 'bmzx': - urljs = 'https://www.suzhou.gov.cn/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=b015bfa5e5514cc9a26cd9f956ef8e69'; - url = 'https://www.suzhou.gov.cn/szsrmzf/bmzx/bmzx_list.shtml'; - title = '苏州市政府 - 民生资讯'; - break; - default: - logger.error('pattern not matched'); - } - if (urljs) { - const responsejs = await got({ - method: 'get', - url: urljs, - Host: 'www.suzhou.gov.cn', - }); - const jsdata = responsejs.data.infolist; - - ctx.state.data = { - title, - link: url, - item: await Promise.all( - jsdata.slice(0, 10).map(async (item) => { - // 获取全文 - const link = item.link.indexOf('http') === -1 ? liburl.resolve(root_url, item.link) : item.link; - const description = await ctx.cache.tryGet(link, async () => { - let responsehtml; - - try { - responsehtml = await got({ - method: 'get', - url: link, - }); - const content = util.content(responsehtml.data); - return content; - } catch (error) { - return ''; - } - }); - return { - title: item.title, - description, - link, - pubDate: item.pubtime, - }; - }) - ), - }; - } - if (!urljs) { - const response = await got({ - method: 'get', - url, - }); - - const $ = cheerio.load(response.data); - const list = $('div.pageList li'); - - ctx.state.data = { - title, - link: url, - item: await Promise.all( - list - .slice(0, 10) - .map((_, item) => { - item = $(item); - const a = item.find('a'); - const link = liburl.resolve(root_url, a.attr('href')); - return { - title: a.attr('title'), - link, - pubDate: new Date(item.find('.time').text()).toUTCString(), - }; - }) - .get() - ), - }; - } -}; diff --git a/lib/routes/gov/suzhou/utils.js b/lib/routes/gov/suzhou/utils.js deleted file mode 100644 index a687caa2783463..00000000000000 --- a/lib/routes/gov/suzhou/utils.js +++ /dev/null @@ -1,20 +0,0 @@ -const cheerio = require('cheerio'); - -const content = (response) => { - const e = cheerio.load(response); - e('.article-extended').remove(); - e('#shareNode').remove(); - e('.article-auxiliary').remove(); - const content = e('.contentShow').html() - ? e('.contentShow').html() - : e('div.detail') - .html() - .replace(/<.?ucaptitle>/g, '') - .replace(/<.?ucapcontent>/g, '') - .replace(/"\/szsrmzf/g, '"https://www.suzhou.gov.cn/szsrmzf') - .replace('publishtime', 'span'); - return content; -}; -module.exports = { - content, -}; diff --git a/lib/v2/gov/maintainer.js b/lib/v2/gov/maintainer.js index e02d783226edf2..ad148d8cd12ab5 100644 --- a/lib/v2/gov/maintainer.js +++ b/lib/v2/gov/maintainer.js @@ -101,6 +101,9 @@ module.exports = { '/shenzhen/zzb/:caty/:page?': ['zlasd'], '/sichuan/deyang/govpublicinfo/:countyName/:infoType?': ['zytomorrow'], '/sichuan/deyang/mztoday/:infoType?': ['zytomorrow'], + '/suzhou/doc': ['EsuRt'], + '/suzhou/fg/:category?': ['nczitzk'], + '/suzhou/news/:uid': ['EsuRt', 'luyuhuang'], '/taiyuan/rsj/:caty/:page?': ['2PoL'], '/wuhan/sy/whyw': ['nczitzk'], '/xinyi/:path+': ['ShuiHuo'], diff --git a/lib/v2/gov/radar.js b/lib/v2/gov/radar.js index 35da7bc7942777..37651a679cca03 100644 --- a/lib/v2/gov/radar.js +++ b/lib/v2/gov/radar.js @@ -2287,6 +2287,31 @@ module.exports = { }, ], }, + 'suzhou.gov.cn': { + _name: '苏州市人民政府', + fg: [ + { + title: '苏州市发展和改革委员会', + docs: 'https://docs.rsshub.app/routes/government#su-zhou-shi-ren-min-zheng-fu-su-zhou-shi-fa-zhan-he-gai-ge-wei-yuan-hui', + source: ['/:category*'], + target: (params) => `/gov/suzhou/fg/${params.replace(/\.shtml/, '')}`, + }, + ], + www: [ + { + title: '政府信息公开文件', + docs: 'https://docs.rsshub.app/routes/government#su-zhou-shi-ren-min-zheng-fu', + source: ['/szxxgk/front/xxgk_right.jsp', '/'], + target: '/gov/suzhou/doc', + }, + { + title: '政府新闻', + docs: 'https://docs.rsshub.app/routes/government#su-zhou-shi-ren-min-zheng-fu', + source: ['/szsrmzf/:uid/nav_list.shtml'], + target: '/gov/suzhou/news/:uid', + }, + ], + }, 'sz.gov.cn': { _name: '深圳政府在线移动门户', hrss: [ diff --git a/lib/v2/gov/router.js b/lib/v2/gov/router.js index 1164beb421f0ee..0d1ec6a01303ac 100644 --- a/lib/v2/gov/router.js +++ b/lib/v2/gov/router.js @@ -93,6 +93,9 @@ module.exports = function (router) { router.get('/shenzhen/zzb/:caty/:page?', require('./shenzhen/zzb/index')); router.get('/sichuan/deyang/govpublicinfo/:countyName/:infoType?', require('./sichuan/deyang/govpublicinfo')); router.get('/sichuan/deyang/mztoday/:infoType?', require('./sichuan/deyang/mztoday')); + router.get('/suzhou/doc', require('./suzhou/doc')); + router.get('/suzhou/fg/:category*', require('./suzhou/fg')); + router.get('/suzhou/news/:uid', require('./suzhou/news')); router.get('/taiyuan/rsj/:caty/:page?', require('./taiyuan/rsj')); router.get('/wuhan/sy/whyw', require('./wuhan/whyw')); router.get(/xinyi(\/[\w/-]+)?/, require('./xinyi/xinyi')); diff --git a/lib/v2/gov/suzhou/doc.js b/lib/v2/gov/suzhou/doc.js new file mode 100644 index 00000000000000..fddf232695fb37 --- /dev/null +++ b/lib/v2/gov/suzhou/doc.js @@ -0,0 +1,46 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const timezone = require('@/utils/timezone'); + +module.exports = async (ctx) => { + const link = 'https://www.suzhou.gov.cn/szxxgk/front/xxgk_right.jsp'; + + const { data: response } = await got(link); + const $ = cheerio.load(response); + const list = $('.tr_main_value_odd') + .toArray() + .map((item) => { + item = $(item); + const title = item.find('a'); + return { + title: title.attr('title'), + link: `https://www.suzhou.gov.cn${title.attr('href')}`, + pubDate: timezone(parseDate(item.find('td:nth-child(3)').text().trim()), 8), + }; + }); + + const items = await Promise.all( + list.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data: response } = await got(item.link); + const $ = cheerio.load(response); + + item.description = $('.article-content').html(); + item.author = $('dd.addWidth:nth-child(3) div').text().trim(); + item.pubDate = $('meta[name="PubDate"]').length ? timezone(parseDate($('meta[name="PubDate"]').attr('content'), 'YYYY-MM-DD HH:mm:ss'), 8) : item.pubDate; + item.category = $('.OwnerDept font') + .toArray() + .map((item) => $(item).text().trim()); + + return item; + }) + ) + ); + + ctx.state.data = { + title: '苏州市政府 - 政策公开文件', + link, + item: items, + }; +}; diff --git a/lib/v2/gov/suzhou/fg.js b/lib/v2/gov/suzhou/fg.js new file mode 100644 index 00000000000000..5e2d7b94173919 --- /dev/null +++ b/lib/v2/gov/suzhou/fg.js @@ -0,0 +1,62 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const timezone = require('@/utils/timezone'); +const { parseDate } = require('@/utils/parse-date'); + +module.exports = async (ctx) => { + const { category = 'szfgw/ggl/nav_list' } = ctx.params; + const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 30; + + const rootUrl = 'https://fg.suzhou.gov.cn'; + const currentUrl = new URL(`${category}.shtml`, rootUrl).href; + + const { data: response } = await got(currentUrl); + + const $ = cheerio.load(response); + + let items = $('h4 a[title]') + .slice(0, limit) + .toArray() + .map((item) => { + item = $(item); + + return { + title: item.prop('title') || item.text(), + link: new URL(item.prop('href'), rootUrl).href, + author: item.find('.author').text(), + pubDate: parseDate(item.parent().find('span.time').text().trim()), + }; + }); + + items = await Promise.all( + items.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data: detailResponse } = await got(item.link); + + const content = cheerio.load(detailResponse); + + item.title = content('ucaptitle').text().trim(); + item.description = content('ucapcontent').html(); + item.author = content('span.ly b').text().trim(); + item.pubDate = timezone(parseDate(content('meta[name="PubDate"]').prop('content')), +8); + + return item; + }) + ) + ); + + const author = $('meta[name="SiteName"]').prop('content'); + const subtitle = $('meta[name="ColumnName"]').prop('content'); + const image = new URL($('div.logo img').prop('src'), rootUrl).href; + + ctx.state.data = { + item: items, + title: `${author} - ${subtitle}`, + link: currentUrl, + description: $('meta[name="ColumnDescription"]').prop('content'), + language: $('html').prop('lang'), + image, + subtitle, + author, + }; +}; diff --git a/lib/v2/gov/suzhou/news.js b/lib/v2/gov/suzhou/news.js new file mode 100644 index 00000000000000..2db427155543b9 --- /dev/null +++ b/lib/v2/gov/suzhou/news.js @@ -0,0 +1,116 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const timezone = require('@/utils/timezone'); + +module.exports = async (ctx) => { + const rootUrl = 'https://www.suzhou.gov.cn'; + const uid = ctx.params.uid; + let url = ''; + let title = ''; + let apiUrl = ''; + let items = []; + switch (uid) { + case 'szyw': + case 'news': + apiUrl = `${rootUrl}/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=5057aeffb1a84a7e8aeded87728da48c`; + url = `${rootUrl}/szsrmzf/szyw/nav_list.shtml`; + title = '苏州市政府 - 苏州要闻'; + break; + case 'qxkx': + case 'district': + apiUrl = `${rootUrl}/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=75c636ea0efb487ea7e479e3cc0ff3e5`; + url = `${rootUrl}/szsrmzf/qxkx/nav_list.shtml`; + title = '苏州市政府 - 区县快讯'; + break; + case 'bmdt': + apiUrl = `${rootUrl}/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=b3d097e3eb79421f88439ea381ce33c3`; + url = `${rootUrl}/szsrmzf/bmdt/nav_list.shtml`; + title = '苏州市政府 - 部门动态'; + break; + case 'xwsp': + apiUrl = `${rootUrl}/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=507980d214c943ebb0a70853ec94b12e`; + url = `${rootUrl}/szsrmzf/xwsp/nav_list.shtml`; + title = '苏州市政府 - 新闻视频'; + break; + case 'rdzt': + url = `${rootUrl}/szsrmzf/rdzt/nav_list.shtml`; + title = '苏州市政府 - 热点专题'; + break; + case 'sbjzt': + url = `${rootUrl}/szsrmzf/sbjzt/nav_list.shtml`; + title = '苏州市政府 - 市本级专题'; + break; + case 'zxrdzt': + url = `${rootUrl}/szsrmzf/zxrdzt/nav_list.shtml`; + title = '苏州市政府 - 最新热点专题'; + break; + case 'wqzt': + url = `${rootUrl}/szsrmzf/wqzt/nav_list.shtml`; + title = '苏州市政府 - 往期专题'; + break; + case 'qxzt': + url = `${rootUrl}/szsrmzf/qxzt/nav_list.shtml`; + title = '苏州市政府 - 区县专题'; + break; + case 'zwgg': + apiUrl = `${rootUrl}/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=260915178a1f4c4fac44c4bf6378c9b0`; + url = `${rootUrl}/szsrmzf/zwgg/nav_list.shtml`; + title = '苏州市政府 - 政务公告'; + break; + case 'mszx': + apiUrl = `${rootUrl}/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=dc60acecb0be46b89d42272dcb8bd32b`; + url = `${rootUrl}/szsrmzf/mszx/nav_list.shtml`; + title = '苏州市政府 - 便民公告'; + break; + case 'bmzx': + apiUrl = `${rootUrl}/szinf/info/getInfoCommon/?pagesize=15&currpage=1&channelid=b015bfa5e5514cc9a26cd9f956ef8e69`; + url = `${rootUrl}/szsrmzf/bmzx/bmzx_list.shtml`; + title = '苏州市政府 - 民生资讯'; + break; + default: + throw 'pattern not matched'; + } + if (apiUrl) { + const response = await got(apiUrl); + const infoList = response.data.infolist.map((item) => ({ + title: item.title, + link: item.link.startsWith('http') ? item.link : new URL(item.link, rootUrl).href, + pubDate: timezone(parseDate(item.pubtime, 'YYYY-MM-DD HH:mm:ss'), 8), + })); + + items = await Promise.all( + infoList.map((item) => + // 获取全文 + ctx.cache.tryGet(item.link, async () => { + const response = await got(item.link); + const $ = cheerio.load(response.data); + item.description = $('ucapcontent').html(); + + return item; + }) + ) + ); + } else { + const response = await got(url); + + const $ = cheerio.load(response.data); + items = $('ul.infolist li') + .toArray() + .map((item) => { + item = $(item); + const a = item.find('a'); + return { + title: a.attr('title'), + link: new URL(a.attr('href'), rootUrl).href, + pubDate: timezone(parseDate(item.find('.time').text(), 'YYYY-MM-DD'), 8), + }; + }); + } + + ctx.state.data = { + title, + link: url, + item: items, + }; +}; diff --git a/website/docs/routes/government.mdx b/website/docs/routes/government.mdx index 9cd45d80a1b452..bb9ae8a4f0adb6 100644 --- a/website/docs/routes/government.mdx +++ b/website/docs/routes/government.mdx @@ -1080,7 +1080,7 @@ ### 政府新闻 {#su-zhou-shi-ren-min-zheng-fu-zheng-fu-xin-wen} - + | 新闻栏目名 | :uid | | :--------: | :--------------: | | 苏州要闻 | news 或 szyw | @@ -1110,7 +1110,15 @@ ### 政府信息公开文件 {#su-zhou-shi-ren-min-zheng-fu-zheng-fu-xin-xi-gong-kai-wen-jian} - + + +### 苏州市发展和改革委员会 {#su-zhou-shi-ren-min-zheng-fu-su-zhou-shi-fa-zhan-he-gai-ge-wei-yuan-hui} + + + | 通知公告 | 发改要闻 | + | ------------------- | -------------------- | + | szfgw/ggl/nav\_list | szfgw/gzdt/nav\_list | + ## 台湾行政院消费者保护会 {#tai-wan-xing-zheng-yuan-xiao-fei-zhe-bao-hu-hui} From a1c0e755420a9057bfdb763ca2910e6c48b4870a Mon Sep 17 00:00:00 2001 From: Tony Date: Sun, 14 Jan 2024 18:11:00 +0000 Subject: [PATCH 5/6] fix(route): tencent author (#14241) --- lib/v2/tencent/news/author.js | 24 ++++++++++++++++++++---- lib/v2/tencent/templates/news/image.art | 4 ++++ 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 lib/v2/tencent/templates/news/image.art diff --git a/lib/v2/tencent/news/author.js b/lib/v2/tencent/news/author.js index 6921419346e3b3..d5fb5bb1605665 100644 --- a/lib/v2/tencent/news/author.js +++ b/lib/v2/tencent/news/author.js @@ -1,8 +1,9 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); -const timezone = require('@/utils/timezone'); const config = require('@/config').value; +const { art } = require('@/utils/render'); +const { join } = require('path'); module.exports = async (ctx) => { const mid = ctx.params.mid; @@ -18,7 +19,7 @@ module.exports = async (ctx) => { const items = await Promise.all( news.map((item) => { const title = item.title; - const pubDate = timezone(parseDate(item.time), +8); + const pubDate = parseDate(item.timestamp, 'X'); const itemUrl = item.url; const author = item.source; const abstract = item.abstract; @@ -26,11 +27,26 @@ module.exports = async (ctx) => { return ctx.cache.tryGet(itemUrl, async () => { const response = await got(itemUrl); const $ = cheerio.load(response.data); - const article = $('#ArticleContent'); + const data = JSON.parse( + $('script:contains("window.DATA")') + .text() + .match(/window\.DATA = (\{.+\});/)[1] + ); + const $data = cheerio.load(data.originContent.text, null, false); + + $data('*') + .contents() + .filter((_, elem) => elem.type === 'comment') + .replaceWith((_, elem) => + art(join(__dirname, '../templates/news/image.art'), { + attribute: elem.data.trim(), + originAttribute: data.originAttribute, + }) + ); return { title, - description: article.html() || abstract, + description: $data.html() || abstract, link: itemUrl, author, pubDate, diff --git a/lib/v2/tencent/templates/news/image.art b/lib/v2/tencent/templates/news/image.art new file mode 100644 index 00000000000000..04120bbbfe2694 --- /dev/null +++ b/lib/v2/tencent/templates/news/image.art @@ -0,0 +1,4 @@ +{{ if attribute?.startsWith('IMG') && originAttribute[attribute] }} + {{ set image = originAttribute[attribute] }} + +{{ /if }} From 4ee41944b8fa8ad6fd9a26814a906ea2de4930f1 Mon Sep 17 00:00:00 2001 From: Gerardyang Date: Mon, 15 Jan 2024 11:23:41 +0800 Subject: [PATCH 6/6] fix: luogu route parse error (#14170) * fix route parse error * use parse-date instead of Date * optimize decode processes * fix typo --------- --- lib/v2/luogu/daily.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/v2/luogu/daily.js b/lib/v2/luogu/daily.js index 39b558b4faa91a..4e8afb5a3efc7c 100644 --- a/lib/v2/luogu/daily.js +++ b/lib/v2/luogu/daily.js @@ -9,19 +9,24 @@ module.exports = async (ctx) => { const $ = cheerio.load(response.data); const title = $('head title').text(); - const firstPost = $('.am-comment-main .am-comment-bd').first(); - const dailyLink = firstPost.find('a').first().attr('href'); - const issueHeading = firstPost.find('h1').text().trim(); + const injectionScript = $('head script:contains("window._feInjection")').text(); + const jsonRaw = injectionScript.match(/window\._feInjection = JSON\.parse\(decodeURIComponent\("(.*?)"\)\);/)[1]; + const jsonDecode = JSON.parse(decodeURIComponent(jsonRaw)); + + const mdRaw = jsonDecode.currentData.post.content; + + const dailyLink = mdRaw.match(/<([^>]*)>/)[1]; const { data: dailyResponse } = await got(dailyLink); const $daily = cheerio.load(dailyResponse); + const issueHeading = $daily('.am-article-title').first().text().trim(); const item = [ { title, description: $daily('#article-content').html(), link, - author: firstPost.find('p').eq(1).text(), + author: jsonDecode.currentData.post.author.name, guid: `${link}#${issueHeading}`, - pubDate: parseDate(issueHeading.match(/(\d{4} 年 \d{1,2} 月 \d{1,2} 日)/)[1], 'YYYY 年 M 月 D 日'), + pubDate: parseDate(jsonDecode.currentData.post.time), }, ];