From 102ef65bba0c9273c1b403624e288831a33c2a30 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Sun, 1 Sep 2024 17:49:54 +0200 Subject: [PATCH] refactor: Improve error management --- bases/ecoindex/cli/app.py | 4 +- components/ecoindex/exceptions/scraper.py | 1 + components/ecoindex/scraper/helper.py | 8 +-- components/ecoindex/scraper/scrap.py | 5 +- development/ui.py | 87 +++++++++++++++-------- 5 files changed, 68 insertions(+), 37 deletions(-) diff --git a/bases/ecoindex/cli/app.py b/bases/ecoindex/cli/app.py index 0059e90..3fec961 100644 --- a/bases/ecoindex/cli/app.py +++ b/bases/ecoindex/cli/app.py @@ -233,9 +233,9 @@ def analyze( logger=logger, ) - for result, success in analysis_results: + for result, error in analysis_results: results.append(result) - if not success: + if error: count_errors += 1 progress.update(task, advance=1) diff --git a/components/ecoindex/exceptions/scraper.py b/components/ecoindex/exceptions/scraper.py index 515d157..24a4e96 100644 --- a/components/ecoindex/exceptions/scraper.py +++ b/components/ecoindex/exceptions/scraper.py @@ -7,5 +7,6 @@ def __init__(self, url: str, status: int, message: str): self.message = message self.url = url self.status = status + self.msg = f"{status}: {message}" pass diff --git a/components/ecoindex/scraper/helper.py b/components/ecoindex/scraper/helper.py index 6bc0661..2044276 100644 --- a/components/ecoindex/scraper/helper.py +++ b/components/ecoindex/scraper/helper.py @@ -12,7 +12,7 @@ def run_page_analysis( wait_after_scroll: int = 3, wait_before_scroll: int = 3, logger=None, -) -> tuple[Result, bool]: +) -> tuple[Result, str | None]: """Run the page analysis and return the result and a boolean indicating if the analysis was successful""" scraper = EcoindexScraper( url=str(url), @@ -22,7 +22,7 @@ def run_page_analysis( page_load_timeout=20, ) try: - return (run(scraper.get_page_analysis()), True) + return (run(scraper.get_page_analysis()), None) except Exception as e: logger.error(f"{url} -- {e.msg if hasattr(e, 'msg') else e}") @@ -36,7 +36,7 @@ def run_page_analysis( nodes=0, requests=0, ), - False, + e.msg if hasattr(e, "msg") else str(e), ) @@ -47,7 +47,7 @@ def bulk_analysis( wait_after_scroll: int = 0, wait_before_scroll: int = 0, logger=None, -) -> Generator[tuple[Result, bool], None, None]: +) -> Generator[tuple[Result, str | None], None, None]: with ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_analysis = {} diff --git a/components/ecoindex/scraper/scrap.py b/components/ecoindex/scraper/scrap.py index aeffe82..89b7338 100644 --- a/components/ecoindex/scraper/scrap.py +++ b/components/ecoindex/scraper/scrap.py @@ -9,6 +9,7 @@ from ecoindex.models.compute import PageMetrics, Result, ScreenShot, WindowSize from ecoindex.models.scraper import MimetypeAggregation, RequestItem, Requests from ecoindex.utils.screenshots import convert_screenshot_to_webp, set_screenshot_rights +from playwright._impl._network import Response from playwright.async_api import async_playwright from typing_extensions import deprecated @@ -72,7 +73,7 @@ async def scrap_page(self) -> PageMetrics: ignore_https_errors=True, ) response = await self.page.goto(self.url) - await self.check_page_response(response) + await self.check_page_response(response) # type: ignore await self.page.wait_for_load_state() sleep(self.wait_before_scroll) @@ -142,7 +143,7 @@ def get_request_size(self, entry) -> int: else: return len(json.dumps(entry["response"]).encode("utf-8")) - async def check_page_response(self, response) -> None: + async def check_page_response(self, response: Response) -> None: if response and response.status != 200: raise EcoindexScraperStatusException( url=self.url, diff --git a/development/ui.py b/development/ui.py index b870bff..9f72c4b 100644 --- a/development/ui.py +++ b/development/ui.py @@ -7,19 +7,21 @@ from loguru import logger st.set_page_config( - page_title="Analyse Ecoindex", - page_icon="🌍", + page_title="Ecoindex", + page_icon="data:image/svg+xml,🟢", initial_sidebar_state="auto", ) -st.title("Analyse Ecoindex") +st.title("Ecoindex") -form_container = st.container() -results_container = st.container() +tab_local, tab_website = st.tabs(["Ecoindex Local", "Ecoindex.fr"]) + +form_container = tab_local.container() +results_container = tab_local.container() +form_container.header("Lancer une analyse") try: import playwright # noqa - st.toast("Playwright est installé", icon="🎉") except ImportError: st.toast("Playwright n'est pas installé", icon="⚠️") st.info("Pour configurer Playwright, veuillez suivre les instructions suivantes:") @@ -51,13 +53,31 @@ def run_analysis( progress_bar = results_container.progress( 0, text=f"{nb_analysis} analyse en cours..." ) - for i, (result, success) in enumerate(analysis_results): + + for i, (result, error) in enumerate(analysis_results): progress_bar.progress((i + 1) / nb_analysis) - analysis.append({"success": success, **result.__dict__}) + analysis.append({**result.__dict__, "error": error}) progress_bar.empty() df = pandas.DataFrame(analysis) + # Apply colors on error column style + # df.style.applymap(lambda x: "color: red" if x else "", subset=["error"]) + + # Apply colors on grade column + colors = { + "A": "#349A47", + "B": "#51B84B", + "C": "#CADB2A", + "D": "#F6EB15", + "E": "#FECD06", + "F": "#F99839", + "G": "#ED2124", + } + # df = df.style.applymap( + # lambda x: f"color: {colors[x]}" if x else "", subset=["grade"] + # ) + results_container.header("Résultats de l'analyse") results_container.dataframe( df, @@ -134,6 +154,10 @@ def run_analysis( help="Type de page", disabled=True, ), + "error": st.column_config.TextColumn( + "Erreur", + help="Erreur lors de l'analyse", + ), }, hide_index=True, ) @@ -153,31 +177,36 @@ def run_analysis( with st.expander("Options"): col1, col2 = st.columns(2) - with col1: - wait_before_scroll = st.number_input( - label="Attendre avant de scroller", - value=3, - min_value=0, - help="Temps d'attente avant de scroller en secondes pour que la page se charge complètement", - ) + wait_before_scroll = col1.number_input( + label="Attendre avant de scroller", + value=3, + min_value=0, + help="Temps d'attente avant de scroller en secondes pour que la page se charge complètement", + ) - with col2: - wait_after_scroll = st.number_input( - label="Attendre après avoir scrollé", - value=3, - min_value=0, - help="Temps d'attente après avoir scrollé en secondes pour que la page se charge complètement", - ) + wait_after_scroll = col2.number_input( + label="Attendre après avoir scrollé", + value=3, + min_value=0, + help="Temps d'attente après avoir scrollé en secondes pour que la page se charge complètement", + ) basic_auth = st.text_input(label="Authentification de base", disabled=True) session_cookie = st.text_input(label="Cookie de session", disabled=True) submitted = st.form_submit_button( label="Lancer l'analyse", - on_click=run_analysis, - kwargs={ - "urls": urls, - "sizes": sizes, - "wait_before_scroll": wait_before_scroll, - "wait_after_scroll": wait_after_scroll, - }, ) + + if submitted: + run_analysis( + urls=urls, + sizes=sizes, + wait_before_scroll=wait_before_scroll, + wait_after_scroll=wait_after_scroll, + ) + + +container_website_search = tab_website.container() +container_website_results = tab_website.container() + +container_website_search.header("Rechercher un résultat d'analyse")