From e3548ec89723ac29286d06ca1d78c03387e5b8dd Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Wed, 31 Jan 2024 20:02:24 +0800 Subject: [PATCH] fix linters --- docs/templates/e-commerce.rst | 2 +- tests/pages/test_product_navigation_heuristics.py | 6 +++--- .../pages/product_navigation_heuristics.py | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/templates/e-commerce.rst b/docs/templates/e-commerce.rst index e2a8684..e13d79c 100644 --- a/docs/templates/e-commerce.rst +++ b/docs/templates/e-commerce.rst @@ -19,6 +19,6 @@ Parameters .. autoenum:: zyte_spider_templates.spiders.ecommerce.EcommerceCrawlStrategy -.. autoenum:: zyte_spider_templates.spiders.ecommerce.ExtractFrom +.. autoenum:: zyte_spider_templates.spiders.base.ExtractFrom .. autoenum:: zyte_spider_templates.spiders.base.Geolocation diff --git a/tests/pages/test_product_navigation_heuristics.py b/tests/pages/test_product_navigation_heuristics.py index a0db58b..9fd4250 100644 --- a/tests/pages/test_product_navigation_heuristics.py +++ b/tests/pages/test_product_navigation_heuristics.py @@ -1,6 +1,6 @@ import pytest from pytest_twisted import ensureDeferred -from web_poet import HttpResponse, PageParams, RequestUrl +from web_poet import AnyResponse, HttpResponse, PageParams, RequestUrl from zyte_common_items import ProbabilityRequest, ProductNavigation from zyte_spider_templates.pages.product_navigation_heuristics import ( @@ -38,7 +38,7 @@ async def test_unknown_product_page(): """ - response = HttpResponse("https://example.com", body) + response = AnyResponse(HttpResponse("https://example.com", body)) navigation = ProductNavigation.from_dict( { "url": "https://example.com", @@ -118,7 +118,7 @@ async def test_crawl_nofollow_links(): """ url = "https://example.com" - response = HttpResponse(url, body) + response = AnyResponse(HttpResponse(url, body)) request_url = RequestUrl(response.url) navigation = ProductNavigation(url=url) diff --git a/zyte_spider_templates/pages/product_navigation_heuristics.py b/zyte_spider_templates/pages/product_navigation_heuristics.py index 2270584..bd012ff 100644 --- a/zyte_spider_templates/pages/product_navigation_heuristics.py +++ b/zyte_spider_templates/pages/product_navigation_heuristics.py @@ -50,7 +50,9 @@ def _probably_category_links(self) -> List[ProbabilityRequest]: ignore_urls = set(self._urls_for_category()) links = [] - response = TextResponse(url=str(self.response.url), body=self.response.text.encode()) + response = TextResponse( + url=str(self.response.url), body=self.response.text.encode() + ) for link in link_extractor.extract_links(response): if link.url in ignore_urls: continue