forked from loveNight/PythonScripts
-
Notifications
You must be signed in to change notification settings - Fork 1
/
百度搜索图片下载脚本.py
243 lines (220 loc) · 8.37 KB
/
百度搜索图片下载脚本.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: loveNight
# @Date: 2015-10-28 19:59:24
# @Last Modified by: loveNight
# @Last Modified time: 2015-11-15 19:43:20
import urllib
import requests
import os
import re
import sys
import time
import threading
from datetime import datetime as dt
from multiprocessing.dummy import Pool
from multiprocessing import Queue
class BaiduImgDownloader(object):
"""百度图片搜索下载工具,目前只支持单个关键词"""
# 解码网址用的映射表
str_table = {
'_z2C$q': ':',
'_z&e3B': '.',
'AzdH3F': '/'
}
char_table = {
'w': 'a',
'k': 'b',
'v': 'c',
'1': 'd',
'j': 'e',
'u': 'f',
'2': 'g',
'i': 'h',
't': 'i',
'3': 'j',
'h': 'k',
's': 'l',
'4': 'm',
'g': 'n',
'5': 'o',
'r': 'p',
'q': 'q',
'6': 'r',
'f': 's',
'p': 't',
'7': 'u',
'e': 'v',
'o': 'w',
'8': '1',
'd': '2',
'n': '3',
'9': '4',
'c': '5',
'm': '6',
'0': '7',
'b': '8',
'l': '9',
'a': '0'
}
re_objURL = re.compile(r'"objURL":"(.*?)".*?"type":"(.*?)"')
re_downNum = re.compile(r"已下载\s(\d+)\s张图片")
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36",
"Accept-Encoding": "gzip, deflate, sdch",
}
def __init__(self, word, dirpath=None, processNum=30):
if " " in word:
raise AttributeError("本脚本仅支持单个关键字")
self.word = word
self.char_table = {ord(key): ord(value)
for key, value in BaiduImgDownloader.char_table.items()}
if not dirpath:
dirpath = os.path.join(sys.path[0], 'results')
self.dirpath = dirpath
self.jsonUrlFile = os.path.join(sys.path[0], 'jsonUrl.txt')
self.logFile = os.path.join(sys.path[0], 'logInfo.txt')
self.errorFile = os.path.join(sys.path[0], 'errorUrl.txt')
if os.path.exists(self.errorFile):
os.remove(self.errorFile)
if not os.path.exists(self.dirpath):
os.mkdir(self.dirpath)
self.pool = Pool(30)
self.session = requests.Session()
self.session.headers = BaiduImgDownloader.headers
self.queue = Queue()
self.messageQueue = Queue()
self.index = 0 # 图片起始编号,牵涉到计数,不要更改
self.promptNum = 10 # 下载几张图片提示一次
self.lock = threading.Lock()
self.delay = 1.5 # 网络请求太频繁会被封
self.QUIT = "QUIT" # Queue中表示任务结束
self.printPrefix = "**" # 用于指定在控制台输出
def start(self):
# 控制台输出线程
t = threading.Thread(target=self.__log)
t.setDaemon(True)
t.start()
self.messageQueue.put(self.printPrefix + "脚本开始执行")
start_time = dt.now()
urls = self.__buildUrls()
self.messageQueue.put(
self.printPrefix + "已获取 %s 个Json请求网址" % len(urls))
# 解析出所有图片网址,该方法会阻塞直到任务完成
self.pool.map(self.__resolveImgUrl, urls)
while self.queue.qsize():
imgs = self.queue.get()
self.pool.map_async(self.__downImg, imgs)
self.pool.close()
self.pool.join()
self.messageQueue.put(self.printPrefix + "下载完成!已下载 %s 张图片,总用时 %s" %
(self.index, dt.now() - start_time))
self.messageQueue.put(self.printPrefix + "请到 %s 查看结果!" % self.dirpath)
self.messageQueue.put(self.printPrefix + "错误信息保存在 %s" % self.errorFile)
self.messageQueue.put(self.QUIT)
def __log(self):
"""控制台输出,加锁以免被多线程打乱"""
with open(self.logFile, "w", encoding="utf-8") as f:
while True:
message = self.messageQueue.get()
if message == self.QUIT:
break
message = str(dt.now()) + " " + message
if self.printPrefix in message:
print(message)
elif "已下载" in message:
# 下载N张图片提示一次
downNum = self.re_downNum.findall(message)
if downNum and int(downNum[0]) % self.promptNum == 0:
print(message)
f.write(message + '\n')
f.flush()
def __getIndex(self):
"""获取文件编号"""
self.lock.acquire()
try:
return self.index
finally:
self.index += 1
self.lock.release()
def decode(self, url):
"""解码图片URL
解码前:
ippr_z2C$qAzdH3FAzdH3Ffl_z&e3Bftgwt42_z&e3BvgAzdH3F4omlaAzdH3Faa8W3ZyEpymRmx3Y1p7bb&mla
解码后:
http://s9.sinaimg.cn/mw690/001WjZyEty6R6xjYdtu88&690
"""
# 先替换字符串
for key, value in self.str_table.items():
url = url.replace(key, value)
# 再替换剩下的字符
return url.translate(self.char_table)
def __buildUrls(self):
"""json请求网址生成器"""
word = urllib.parse.quote(self.word)
url = r"http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&fp=result&queryWord={word}&cl=2&lm=-1&ie=utf-8&oe=utf-8&st=-1&ic=0&word={word}&face=0&istype=2nc=1&pn={pn}&rn=60"
time.sleep(self.delay)
html = self.session.get(
url.format(word=word, pn=0), timeout=15).content.decode('utf-8')
results = re.findall(r'"displayNum":(\d+),', html)
maxNum = int(results[0]) if results else 0
urls = [url.format(word=word, pn=x)
for x in range(0, maxNum + 1, 60)]
with open(self.jsonUrlFile, "w", encoding="utf-8") as f:
for url in urls:
f.write(url + "\n")
return urls
def __resolveImgUrl(self, url):
"""从指定网页中解析出图片URL"""
time.sleep(self.delay)
html = self.session.get(url, timeout=15).content.decode('utf-8')
datas = self.re_objURL.findall(html)
imgs = [Image(self.decode(x[0]), x[1]) for x in datas]
self.messageQueue.put(self.printPrefix + "已解析出 %s 个图片网址" % len(imgs))
self.queue.put(imgs)
def __downImg(self, img):
"""下载单张图片,传入的是Image对象"""
imgUrl = img.url
# self.messageQueue.put("线程 %s 正在下载 %s " %
# (threading.current_thread().name, imgUrl))
try:
time.sleep(self.delay)
res = self.session.get(imgUrl, timeout=15)
message = None
if str(res.status_code)[0] == "4":
message = "\n%s: %s" % (res.status_code, imgUrl)
elif "text/html" in res.headers["Content-Type"]:
message = "\n无法打开图片: %s" % imgUrl
except Exception as e:
message = "\n抛出异常: %s\n%s" % (imgUrl, str(e))
finally:
if message:
self.messageQueue.put(message)
self.__saveError(message)
return
index = self.__getIndex()
# index从0开始
self.messageQueue.put("已下载 %s 张图片:%s" % (index + 1, imgUrl))
filename = os.path.join(self.dirpath, str(index) + "." + img.type)
with open(filename, "wb") as f:
f.write(res.content)
def __saveError(self, message):
self.lock.acquire()
try:
with open(self.errorFile, "a", encoding="utf-8") as f:
f.write(message)
finally:
self.lock.release()
class Image(object):
"""图片类,保存图片信息"""
def __init__(self, url, type):
super(Image, self).__init__()
self.url = url
self.type = type
if __name__ == '__main__':
print("欢迎使用百度图片下载脚本!\n目前仅支持单个关键词。")
print("下载结果保存在脚本目录下的results文件夹中。")
print("=" * 50)
word = input("请输入你要下载的图片关键词:\n")
down = BaiduImgDownloader(word)
down.start()