feat: 🎸 bing 多区域

alisonjoe · Feb 18, 2024 · aa7e9d0 · aa7e9d0
1 parent a6d3537
commit aa7e9d0
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 8 deletions.
diff --git a/wallpaper_scrapy/@eaDir/.DS_Store@SynoResource b/wallpaper_scrapy/@eaDir/.DS_Store@SynoResource
diff --git a/wallpaper_scrapy/wallpaper_scrapy/hugomiddlewares.py b/wallpaper_scrapy/wallpaper_scrapy/hugomiddlewares.py
@@ -1,7 +1,9 @@
 import os
+import pymysql
 import subprocess
 from scrapy import signals
 from scrapy.utils.project import get_project_settings
+from wallpaper_scrapy.mariadb import DBConnectionPool
 from jinja2 import Template
 
 
@@ -10,11 +12,11 @@ class HugoMiddleware:
     def from_crawler(cls, crawler):
         middleware = cls()
         crawler.signals.connect(middleware.files_downloaded, signal=signals.item_scraped)
-        return middleware   
+        return middleware
 
     def files_downloaded(self, item, response, spider):
         # 在每次下载完成后触发
-        self.hugo_gen_html(item, spider)  
+        self.hugo_gen_html(item, spider)
 
     def hugo_gen_html(self, item, spider):
         settings = get_project_settings()
@@ -23,6 +25,13 @@ def hugo_gen_html(self, item, spider):
         self.hugo_gen_toml(item, working_directory)
         self.hugo_gen(working_directory)
         self.push_github(working_directory)
+        try:
+            db_pool = DBConnectionPool()
+            db_pool.insert_wallpaper(item['trivia_id'], item['image_urls'],
+                                     item['mkt'], item['platform'])
+        except pymysql.Error as e:
+            # 打印异常信息或者记录日志
+            spider.logger.error(f"Error: {e}")
 
     def hugo_gen_toml(self, item, cwd_path):
         # 读取 TOML 模板文件
@@ -54,4 +63,3 @@ def push_github(self, cwd_path):
         subprocess.run(["git", "add", "."], cwd=cwd_path)
         subprocess.run(["git", "commit", "-m", "auto add wallpaper"], cwd=cwd_path)
         subprocess.run(["git", "push", "origin"], cwd=cwd_path)
-
diff --git a/wallpaper_scrapy/wallpaper_scrapy/items.py b/wallpaper_scrapy/wallpaper_scrapy/items.py
@@ -16,4 +16,5 @@ class BingScrapyItem(scrapy.Item):
     image_urls = scrapy.Field()
     image_paths = scrapy.Field()
     md_path = scrapy.Field()
-    trivia_id = scrapy.Field()
+    trivia_id = scrapy.Field()
+    mkt = scrapy.Field()
diff --git a/wallpaper_scrapy/wallpaper_scrapy/markdownmiddlewares.py b/wallpaper_scrapy/wallpaper_scrapy/markdownmiddlewares.py
@@ -4,18 +4,20 @@
 from datetime import datetime, timezone, timedelta
 from scrapy.utils.project import get_project_settings
 
+
 class MarkdownMiddleware:
+
     @classmethod
     def from_crawler(cls, crawler):
         middleware = cls()
         crawler.signals.connect(middleware.files_downloaded, signal=signals.item_scraped)
-        return middleware   
+        return middleware
 
     def files_downloaded(self, item, response, spider):
         # 在每次下载完成后触发
-        self.create_md(item, spider)  
+        self.create_md(item, spider)
+
 
-
     def create_md(self, item, spider):
 
         # 获取当前时间
@@ -40,4 +42,4 @@ def create_md(self, item, spider):
 ![{item['trivia_id']}](/{item['platform']}/{item['trivia_id']}.jpg)"""
         print(md_filename)
         with open(md_filename, 'w') as f:
-            f.write(markdown_content)
+            f.write(markdown_content)