From ee1e80f6007d8968cac5ca776856557e707b97a4 Mon Sep 17 00:00:00 2001 From: alisonjoe Date: Sun, 18 Feb 2024 02:45:38 +0800 Subject: [PATCH] feat: init --- wallpaper_scrapy/scrapy.cfg | 2 +- wallpaper_scrapy/wallpaper_scrapy/hugomiddlewares.py | 4 ++-- wallpaper_scrapy/wallpaper_scrapy/settings.py | 12 ++++++------ wallpaper_scrapy/wallpaper_scrapy/spiders/bing.py | 3 ++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/wallpaper_scrapy/scrapy.cfg b/wallpaper_scrapy/scrapy.cfg index 8ffad5502..72121e4db 100644 --- a/wallpaper_scrapy/scrapy.cfg +++ b/wallpaper_scrapy/scrapy.cfg @@ -6,6 +6,6 @@ [settings] default = wallpaper_scrapy.settings -[deploy] +[deploy:wallpaper] url = http://192.168.1.105:6800/ project = wallpaper_scrapy diff --git a/wallpaper_scrapy/wallpaper_scrapy/hugomiddlewares.py b/wallpaper_scrapy/wallpaper_scrapy/hugomiddlewares.py index 4edf8989d..7fa7ac7f5 100644 --- a/wallpaper_scrapy/wallpaper_scrapy/hugomiddlewares.py +++ b/wallpaper_scrapy/wallpaper_scrapy/hugomiddlewares.py @@ -21,8 +21,8 @@ def hugo_gen_html(self, item, spider): working_directory = settings.get('WALLPAPER_BASE') self.hugo_gen_toml(item, working_directory) - #self.hugo_gen(working_directory) - #self.push_github(working_directory) + self.hugo_gen(working_directory) + self.push_github(working_directory) def hugo_gen_toml(self, item, cwd_path): # 读取 TOML 模板文件 diff --git a/wallpaper_scrapy/wallpaper_scrapy/settings.py b/wallpaper_scrapy/wallpaper_scrapy/settings.py index b8db89470..139a554ac 100644 --- a/wallpaper_scrapy/wallpaper_scrapy/settings.py +++ b/wallpaper_scrapy/wallpaper_scrapy/settings.py @@ -61,12 +61,12 @@ 'wallpaper_scrapy.pipelines.WallpaperImgDownloadPipeline': 300 } -#IMAGES_STORE="/app/wallpaper/static" -#BLOG_STORE="/app/wallpaper/content/blog" -#WALLPAPER_BASE="/app/wallpaper" -WALLPAPER_BASE="/Users/alison/WorkSpace/Wallpaper/" -IMAGES_STORE="/Users/alison/WorkSpace/Wallpaper/static" -BLOG_STORE="/Users/alison/WorkSpace/Wallpaper/content/blog" +WALLPAPER_BASE="/app/Wallpaper" +IMAGES_STORE="/app/Wallpaper/static" +BLOG_STORE="/app/Wallpaper/content/blog" +#WALLPAPER_BASE="/Users/alison/WorkSpace/Wallpaper/" +#IMAGES_STORE="/Users/alison/WorkSpace/Wallpaper/static" +#BLOG_STORE="/Users/alison/WorkSpace/Wallpaper/content/blog" # Set settings whose default value is deprecated to a future-proof value REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" diff --git a/wallpaper_scrapy/wallpaper_scrapy/spiders/bing.py b/wallpaper_scrapy/wallpaper_scrapy/spiders/bing.py index 0ec9f43a6..a53a3e05a 100644 --- a/wallpaper_scrapy/wallpaper_scrapy/spiders/bing.py +++ b/wallpaper_scrapy/wallpaper_scrapy/spiders/bing.py @@ -19,7 +19,8 @@ class WallpaperBingSpider(scrapy.Spider): def start_requests(self): for url in self.start_urls: - yield SplashRequest(url=url, callback=self.parse) + #yield SplashRequest(url=url, callback=self.parse, args={'wait': 6, 'timeout':60}) + yield Request(url=url, callback=self.parse) def parse(self, response):