Merge pull request #274 from yldoctrine/some_website_have_no_title

Add robustness for pages missing title
fhamborg · Jul 8, 2024 · 3896c7d · 3896c7d
2 parents d79ae58 + 3ec0c35
commit 3896c7d
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/newsplease/helper_classes/parse_crawler.py b/newsplease/helper_classes/parse_crawler.py
@@ -66,8 +66,8 @@ def pass_to_pipeline(
         article['download_date'] = timestamp
         article['source_domain'] = source_domain.encode("utf-8")
         article['url'] = response.url
-        article['html_title'] = response.selector.xpath('//title/text()') \
-            .extract_first().encode("utf-8")
+        extracted_title = response.selector.xpath('//title/text()').extract_first()
+        article['html_title'] = extracted_title.encode("utf-8") if extracted_title is not None else ''
         if rss_title is None:
             article['rss_title'] = 'NULL'
         else: