From d8f298571605b9a552d4c2c938cac12048955b7c Mon Sep 17 00:00:00 2001 From: julien Date: Fri, 15 Nov 2024 22:23:15 +0100 Subject: [PATCH] refactor: use async for httpx requests --- .../management/commands/check_next_article.py | 3 +- backend/caviardeul/services/articles.py | 45 ++++++++++--------- backend/caviardeul/views/custom_article.py | 6 +-- backend/caviardeul/views/daily_article.py | 2 +- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/backend/caviardeul/management/commands/check_next_article.py b/backend/caviardeul/management/commands/check_next_article.py index cc1d4e1..7ef6db6 100644 --- a/backend/caviardeul/management/commands/check_next_article.py +++ b/backend/caviardeul/management/commands/check_next_article.py @@ -1,3 +1,4 @@ +from asgiref.sync import async_to_sync from django.core.management.base import BaseCommand, CommandError from django.utils import timezone @@ -16,6 +17,6 @@ def handle(self, *args, **options): raise CommandError("No daily article left") try: - get_article_html_from_wikipedia(next_article.page_id) + async_to_sync(get_article_html_from_wikipedia)(next_article.page_id) except ArticleFetchError: raise CommandError("Error when retrieving daily article") diff --git a/backend/caviardeul/services/articles.py b/backend/caviardeul/services/articles.py index eb4cb41..0bdd644 100644 --- a/backend/caviardeul/services/articles.py +++ b/backend/caviardeul/services/articles.py @@ -11,47 +11,48 @@ from caviardeul.services.parsing import strip_html_article -def get_article_content(article: Article) -> str: - content = _get_article_content_from_cache(article.page_id) +async def get_article_content(article: Article) -> str: + content = await _get_article_content_from_cache(article.page_id) if content is not None: logger.debug("retrieved article from cache", extra={"page_id": article.page_id}) return content - _, html_content = get_article_html_from_wikipedia(article.page_id) + _, html_content = await get_article_html_from_wikipedia(article.page_id) logger.info("retrieved article from wikipedia", extra={"page_id": article.page_id}) article_content = _prepare_article_content_from_html( article.page_name, html_content ) - _set_article_to_cache(article.page_id, article_content) + await _set_article_to_cache(article.page_id, article_content) return article_content -def _get_article_content_from_cache(page_id: str) -> str | None: - return cache.get(f"wikipedia::{page_id}") +async def _get_article_content_from_cache(page_id: str) -> str | None: + return await cache.aget(f"wikipedia::{page_id}") -def _set_article_to_cache(page_id: str, content: str) -> None: +async def _set_article_to_cache(page_id: str, content: str) -> None: now = timezone.now() tomorrow = (now + timedelta(days=1)).replace( hour=0, minute=0, second=0, microsecond=0 ) cache_timeout = int((tomorrow - now).total_seconds()) - cache.set(f"wikipedia::{page_id}", content, timeout=cache_timeout) - - -def get_article_html_from_wikipedia(page_id: str) -> tuple[str, str]: - response = httpx.get( - "https://fr.wikipedia.org/w/api.php", - params={ - "action": "parse", - "format": "json", - "prop": "text", - "formatversion": 2, - "origin": "*", - "page": page_id, - }, - ) + await cache.aset(f"wikipedia::{page_id}", content, timeout=cache_timeout) + + +async def get_article_html_from_wikipedia(page_id: str) -> tuple[str, str]: + async with httpx.AsyncClient() as client: + response = await client.get( + "https://fr.wikipedia.org/w/api.php", + params={ + "action": "parse", + "format": "json", + "prop": "text", + "formatversion": 2, + "origin": "*", + "page": page_id, + }, + ) if response.status_code != 200: raise ArticleFetchError(f"Unexected response from API: {response.status_code}") diff --git a/backend/caviardeul/views/custom_article.py b/backend/caviardeul/views/custom_article.py index 249555e..cc9d1ef 100644 --- a/backend/caviardeul/views/custom_article.py +++ b/backend/caviardeul/views/custom_article.py @@ -25,7 +25,7 @@ async def get_custom_article(request: HttpRequest, public_id: str) -> CustomArti article = await aget_object_or_404(CustomArticle, public_id=public_id) try: - content = get_article_content(article) + content = await get_article_content(article) except ArticleFetchError: raise HttpError(400, "L'article n'a pas été trouvé") @@ -42,7 +42,7 @@ async def create_custom_article( request: HttpRequest, payload: CustomArticleCreateSchema, response: HttpResponse ) -> CustomArticle: try: - page_title, _ = get_article_html_from_wikipedia(payload.page_id) + page_title, _ = await get_article_html_from_wikipedia(payload.page_id) except ArticleFetchError: raise HttpError(400, "L'article n'a pas été trouvé") @@ -63,5 +63,5 @@ async def create_custom_article( ), ) - article.content = get_article_content(article) + article.content = await get_article_content(article) return article diff --git a/backend/caviardeul/views/daily_article.py b/backend/caviardeul/views/daily_article.py index bcbcbaa..6eda4b0 100644 --- a/backend/caviardeul/views/daily_article.py +++ b/backend/caviardeul/views/daily_article.py @@ -83,7 +83,7 @@ async def _get_daily_article_response(queryset: QuerySet[DailyArticle]): return 404, {"detail": "L'article n'a pas été trouvé"} try: - article.content = get_article_content(article) + article.content = await get_article_content(article) except ArticleFetchError: logger.exception( "Error encountered with daily article", extra={"article_id": article.id}