diff --git a/requirements.txt b/requirements.txt index b9fb14a..64fb16a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -Scrapy==2.11.2 -scrapy-zyte-api[provider]>=0.22.1 -zyte-spider-templates==0.11.1 -duplicate-url-discarder[rules]>=0.2.0 +duplicate-url-discarder[rules]>=0.3.0 +scrapinghub[msgpack] # zyte-spider-templates dep, use [msgpack] for speed. +Scrapy==2.12.0 +scrapy-zyte-api[provider]>=0.25.2 +zyte-spider-templates==0.11.2 diff --git a/scrapinghub.yml b/scrapinghub.yml index 7b1e5a7..5fc6a3b 100644 --- a/scrapinghub.yml +++ b/scrapinghub.yml @@ -5,6 +5,6 @@ #projects: # default: NNNNNN stacks: - default: scrapy:2.11-20241022 + default: scrapy:2.12-20241202 requirements: file: requirements.txt diff --git a/zyte_spider_templates_project/settings.py b/zyte_spider_templates_project/settings.py index a184efa..e482688 100644 --- a/zyte_spider_templates_project/settings.py +++ b/zyte_spider_templates_project/settings.py @@ -1,4 +1,7 @@ +from logging import INFO + from itemadapter import ItemAdapter +from scrapy.logformatter import LogFormatter, DROPPEDMSG from zyte_common_items import ZyteItemAdapter ItemAdapter.ADAPTER_CLASSES.appendleft(ZyteItemAdapter) @@ -25,6 +28,22 @@ "scrapy_poet.RetryMiddleware": 275, } + +class CustomLogFormatter(LogFormatter): + def dropped(self, item, exception, response, spider): + return { + 'level': INFO, + 'msg': DROPPEDMSG, + 'args': { + 'exception': exception, + 'item': item, + } + } + + +LOG_FORMATTER = CustomLogFormatter + + # scrapy-poet SCRAPY_POET_DISCOVER = [ "zyte_spider_templates_project.pages",