Skip to content

协程运行速度比多进程慢 #18

Open
@edmund-zhao

Description

@edmund-zhao
@trip.coroutine
def downloadImg():
    if not os.path.exists('./data'):
        os.mkdir('./data')
    q = 0
    for i in downUrl:
#        if q < 1100:
#        	q += 1
#        	continue
        imgUrl = []
        lenOfData = 0
        r = yield s.get(i[0])
        Soup = BeautifulSoup(r.text,'html.parser')
        u = Soup.select('#main > article > div > div')
        if not os.path.exists('./data/{}'.format(i[-1])):
            os.mkdir('./data/{}'.format(i[-1]))
        else:
            lenOfData = len(os.listdir('./data/{}'.format(i[-1])))
        for t in u:
            if t.select('a') == []:
                continue
            imgUrl.append(t.select('a')[0].get('href'))
        if len(imgUrl) == lenOfData:
            print("跳过< {} >".format(i[-1]))
            continue
        t = 0
        for url in imgUrl:
            r = yield s.get(url,stream=True)
            with open('./data/{}/{}.jpg'.format(i[-1],t), 'wb') as fd:
                for chunk in r.iter_content(256):
                    chunk = yield chunk
                    fd.write(chunk)
            print("正在下载<{}>的第{}张图片".format(i[-1],t))
            t += 1
  • 函数如上,全局变量downUrl 是一个数组,保存Url连接的数组

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions