Skip to content

Commit

Permalink
fix(scraper): Fix bulk analyzis helper to return a sync Generator
Browse files Browse the repository at this point in the history
  • Loading branch information
vvatelot committed Aug 30, 2024
1 parent 7525267 commit 16c9209
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 13 deletions.
17 changes: 7 additions & 10 deletions bases/ecoindex/cli/app.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
from asyncio import run
from datetime import datetime
from multiprocessing import cpu_count
from os.path import dirname
Expand Down Expand Up @@ -225,15 +224,13 @@ def analyze(
count_errors = 0
task = progress.add_task("Processing", total=len(urls) * len(window_sizes))

analysis_results = run(
bulk_analysis(
max_workers=max_workers,
urls=urls,
window_sizes=window_sizes,
wait_after_scroll=wait_after_scroll,
wait_before_scroll=wait_before_scroll,
logger=logger,
)
analysis_results = bulk_analysis(
max_workers=max_workers,
urls=urls,
window_sizes=window_sizes,
wait_after_scroll=wait_after_scroll,
wait_before_scroll=wait_before_scroll,
logger=logger,
)

for result, success in analysis_results:
Expand Down
6 changes: 3 additions & 3 deletions components/ecoindex/scraper/helper.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from asyncio import run
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import AsyncGenerator
from typing import Generator

from ecoindex.models.compute import Result, WindowSize
from ecoindex.scraper.scrap import EcoindexScraper
Expand Down Expand Up @@ -40,14 +40,14 @@ def run_page_analysis(
)


async def bulk_analysis(
def bulk_analysis(
max_workers,
urls,
window_sizes,
wait_after_scroll: int = 0,
wait_before_scroll: int = 0,
logger=None,
) -> AsyncGenerator[tuple[Result, bool], None]:
) -> Generator[tuple[Result, bool], None]:
with ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_analysis = {}

Expand Down

0 comments on commit 16c9209

Please sign in to comment.