Write error response to WACZ (#19, PR#20)

scrapy_webarchive/extensions.py

@@ -107,7 +107,7 @@ def from_crawler(cls, crawler: Crawler) -> Self:
         except AttributeError:
             exporter = cls(crawler.settings, crawler)
 
-        crawler.signals.connect(exporter.response_received, signal=signals.response_received)
+        crawler.signals.connect(exporter.response_downloaded, signal=signals.response_downloaded)
         crawler.signals.connect(exporter.spider_closed, signal=signals.spider_closed)
         crawler.signals.connect(exporter.spider_opened, signal=signals.spider_opened)
         return exporter
@@ -143,7 +143,7 @@ def from_settings(cls, settings: Settings, crawler: Crawler):
     def spider_opened(self) -> None:
         self.writer.write_warcinfo(robotstxt_obey=self.settings["ROBOTSTXT_OBEY"])
 
-    def response_received(self, response: Response, request: Request, spider: Spider) -> None:
+    def response_downloaded(self, response: Response, request: Request, spider: Spider) -> None:
         request.meta["WARC-Date"] = get_formatted_dt_string(format=WARC_DT_FORMAT)
 
         # Write response WARC record

tests/test_extensions.py

@@ -40,7 +40,7 @@ def test_get_store(self, *args):
         extension = WaczExporter.from_crawler(crawler)
         assert isinstance(extension.store, FTPFilesStore)
 
-    def test_response_received(self):
+    def test_response_downloaded(self):
         crawler = get_crawler(settings_dict={"SW_EXPORT_URI": "/tmp/scrapy-webarchive/wacz/"})
         crawler.spider = crawler._create_spider("quotes")
         extension = WaczExporter.from_crawler(crawler)
@@ -49,7 +49,7 @@ def test_response_received(self):
         # Call the method under test
         request = Request("http://example.com")
         response = Response(request.url)
-        extension.response_received(response, request, crawler.spider)
+        extension.response_downloaded(response, request, crawler.spider)
 
         # Verify that the WARC date was set in request meta
         assert "WARC-Date" in request.meta