Skip to content

Commit

Permalink
fix(route/apnews): Add throttle to topics.
Browse files Browse the repository at this point in the history
 (#18106)

* fix(route/apnews): Add throttle to topics.

* Update topics.ts
  • Loading branch information
dzx-dzx authored Jan 13, 2025
1 parent 9893318 commit 4208e87
Showing 1 changed file with 11 additions and 12 deletions.
23 changes: 11 additions & 12 deletions lib/routes/apnews/topics.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Route, ViewType } from '@/types';
import got from '@/utils/got';
import { load } from 'cheerio';
import { fetchArticle, removeDuplicateByKey } from './utils';
import { asyncPoolAll, fetchArticle, removeDuplicateByKey } from './utils';
const HOME_PAGE = 'https://apnews.com';

export const route: Route = {
Expand Down Expand Up @@ -41,17 +41,16 @@ async function handler(ctx) {
const response = await got(url);
const $ = load(response.data);

const items = await Promise.all(
$(':is(.PagePromo-content, .PageListStandardE-leadPromo-info) bsp-custom-headline')
.toArray()
.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : Infinity)
.map((e) => ({
title: $(e).find('span.PagePromoContentIcons-text').text(),
link: $(e).find('a').attr('href'),
}))
.filter((e) => typeof e.link === 'string')
.map((item) => (ctx.req.query('fulltext') === 'true' ? fetchArticle(item) : item))
);
const list = $(':is(.PagePromo-content, .PageListStandardE-leadPromo-info) bsp-custom-headline')
.toArray()
.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : Infinity)
.map((e) => ({
title: $(e).find('span.PagePromoContentIcons-text').text(),
link: $(e).find('a').attr('href'),
}))
.filter((e) => typeof e.link === 'string');

const items = ctx.req.query('fulltext') === 'true' ? await asyncPoolAll(10, list, (item) => fetchArticle(item)) : list;

return {
title: $('title').text(),
Expand Down

0 comments on commit 4208e87

Please sign in to comment.