Skip to content

Commit 83d21d5

Browse files
zhanghe06zhanghe06
committed
并发性能测试
1 parent 305947f commit 83d21d5

6 files changed

+279
-3
lines changed

‎cores.py

+130-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
# encoding: utf-8
22
__author__ = 'zhanghe'
33

4-
from multiprocessing import Pool
4+
# from multiprocessing import Pool
5+
from multiprocessing.dummy import Pool
6+
import gevent
57
import os
68
import time
79
import random
10+
from tools import time_log
11+
from gevent import monkey
12+
monkey.patch_all()
813

914

1015
def long_time_task(name):
@@ -15,12 +20,14 @@ def long_time_task(name):
1520
"""
1621
print 'Run task %s (%s)...' % (name, os.getpid())
1722
start = time.time()
18-
time.sleep(random.random() * 3)
23+
# time.sleep(random.random() * 3)
24+
time.sleep(0.3)
1925
end = time.time()
2026
print 'Task %s runs %0.2f seconds.' % (name, (end - start))
2127

2228

23-
if __name__ == '__main__':
29+
@time_log.time_log
30+
def run():
2431
print 'Parent process %s.' % os.getpid()
2532
p = Pool()
2633
for i in range(9):
@@ -31,6 +38,15 @@ def long_time_task(name):
3138
print 'All sub processes done.'
3239

3340

41+
@time_log.time_log
42+
def run_gevent():
43+
threads = [gevent.spawn(long_time_task, i) for i in range(9)]
44+
gevent.joinall(threads)
45+
46+
if __name__ == '__main__':
47+
# run()
48+
run_gevent()
49+
3450
# 这是一个多进程的例子，同时运行的进程数与机器的核心数是对应的
3551
# 根据任务池同时运行的进程数也可以看出当前cpu的核心数，运行结果：
3652
# 可以看出进程池子最多有4个进程
@@ -76,3 +92,114 @@ def long_time_task(name):
7692
# cpu cores : 4
7793
# cpuid level : 13
7894
# zhanghe@ubuntu:~$
95+
96+
97+
# from multiprocessing import Pool
98+
# time.sleep(0.3)
99+
# 方法run开始时间：Thu Apr 30 00:48:04 2015
100+
# Parent process 17113.
101+
# Waiting for all sub processes done...
102+
# Run task 1 (17115)...
103+
# Run task 0 (17114)...
104+
# Task 1 runs 0.30 seconds.
105+
# Run task 2 (17115)...
106+
# Task 0 runs 0.30 seconds.
107+
# Run task 3 (17114)...
108+
# Task 2 runs 0.30 seconds.
109+
# Run task 4 (17115)...
110+
# Task 3 runs 0.30 seconds.
111+
# Run task 5 (17114)...
112+
# Task 4 runs 0.30 seconds.
113+
# Run task 6 (17115)...
114+
# Task 5 runs 0.30 seconds.
115+
# Run task 7 (17114)...
116+
# Task 6 runs 0.30 seconds.
117+
# Run task 8 (17115)...
118+
# Task 7 runs 0.30 seconds.
119+
# Task 8 runs 0.30 seconds.
120+
# All sub processes done.
121+
# 方法run结束时间：Thu Apr 30 00:48:05 2015
122+
# 方法run运行时间：1.72S
123+
124+
125+
# from multiprocessing.dummy import Pool
126+
# time.sleep(0.3)
127+
# 方法run开始时间：Thu Apr 30 00:51:27 2015
128+
# Parent process 17244.
129+
# Waiting for all sub processes done...
130+
# Run task 0 (17244)...
131+
# Run task 1 (17244)...
132+
# Task 0 runs 0.30 seconds.
133+
# Task 1 runs 0.30 seconds.Run task 2 (17244)...
134+
#
135+
# Run task 3 (17244)...
136+
# Task 2 runs 0.30 seconds.Task 3 runs 0.30 seconds.
137+
# Run task 4 (17244)...
138+
#
139+
# Run task 5 (17244)...
140+
# Task 4 runs 0.30 seconds.
141+
# Task 5 runs 0.30 seconds.Run task 6 (17244)...
142+
#
143+
# Run task 7 (17244)...
144+
# Task 6 runs 0.30 seconds.
145+
# Run task 8 (17244)...
146+
# Task 7 runs 0.30 seconds.
147+
# Task 8 runs 0.30 seconds.
148+
# All sub processes done.
149+
# 方法run结束时间：Thu Apr 30 00:51:29 2015
150+
# 方法run运行时间：1.54S
151+
152+
153+
# import gevent
154+
# time.sleep(0.3)
155+
# 方法run_gevent开始时间：Thu Apr 30 01:13:52 2015
156+
# Run task 0 (17964)...
157+
# Task 0 runs 0.30 seconds.
158+
# Run task 1 (17964)...
159+
# Task 1 runs 0.30 seconds.
160+
# Run task 2 (17964)...
161+
# Task 2 runs 0.30 seconds.
162+
# Run task 3 (17964)...
163+
# Task 3 runs 0.30 seconds.
164+
# Run task 4 (17964)...
165+
# Task 4 runs 0.30 seconds.
166+
# Run task 5 (17964)...
167+
# Task 5 runs 0.30 seconds.
168+
# Run task 6 (17964)...
169+
# Task 6 runs 0.30 seconds.
170+
# Run task 7 (17964)...
171+
# Task 7 runs 0.30 seconds.
172+
# Run task 8 (17964)...
173+
# Task 8 runs 0.30 seconds.
174+
# 方法run_gevent结束时间：Thu Apr 30 01:13:55 2015
175+
# 方法run_gevent运行时间：2.74S
176+
177+
178+
# import gevent
179+
# time.sleep(0.3)
180+
# from gevent import monkey
181+
# monkey.patch_all()
182+
# 方法run_gevent开始时间：Thu Apr 30 01:17:05 2015
183+
# Run task 0 (18038)...
184+
# Run task 1 (18038)...
185+
# Run task 2 (18038)...
186+
# Run task 3 (18038)...
187+
# Run task 4 (18038)...
188+
# Run task 5 (18038)...
189+
# Run task 6 (18038)...
190+
# Run task 7 (18038)...
191+
# Run task 8 (18038)...
192+
# Task 0 runs 0.30 seconds.
193+
# Task 1 runs 0.30 seconds.
194+
# Task 2 runs 0.30 seconds.
195+
# Task 3 runs 0.30 seconds.
196+
# Task 4 runs 0.30 seconds.
197+
# Task 5 runs 0.30 seconds.
198+
# Task 6 runs 0.30 seconds.
199+
# Task 7 runs 0.30 seconds.
200+
# Task 8 runs 0.30 seconds.
201+
# 方法run_gevent结束时间：Thu Apr 30 01:17:05 2015
202+
# 方法run_gevent运行时间：0.32S
203+
204+
# 结果可以看出使用multiprocessing.dummy模块比multiprocessing速度略有提升
205+
# 使用打补丁后的gevent，效率最快

‎gevent.py renamed to ‎gevent_test.py

File renamed without changes.

‎performance_testing.py

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# encoding: utf-8
2+
__author__ = 'zhanghe'
3+
4+
import urllib2
5+
from multiprocessing.dummy import Pool as ThreadPool
6+
from tools import time_log
7+
8+
urls = [
9+
'http://www.python.org',
10+
'http://www.python.org/about/',
11+
'http://www.onlamp.com/pub/a/python/2003/04/17/metaclasses.html',
12+
'http://www.python.org/doc/',
13+
'http://www.python.org/download/',
14+
'http://www.python.org/getit/',
15+
'http://www.python.org/community/',
16+
'https://wiki.python.org/moin/',
17+
'http://planet.python.org/',
18+
'https://wiki.python.org/moin/LocalUserGroups',
19+
'http://www.python.org/psf/',
20+
'http://docs.python.org/devguide/',
21+
'http://www.python.org/community/awards/'
22+
# etc..
23+
]
24+
25+
26+
@time_log.time_log
27+
def work(num=2):
28+
# Make the Pool of workers
29+
pool = ThreadPool(num)
30+
# Open the urls in their own threads
31+
# and return the results
32+
results = pool.map(urllib2.urlopen, urls)
33+
# close the pool and wait for the work to finish
34+
pool.close()
35+
pool.join()
36+
37+
38+
@time_log.time_log
39+
def work2(num=2):
40+
p = ThreadPool(num)
41+
for i in urls:
42+
p.apply_async(urllib2.urlopen, args=(i,))
43+
p.close()
44+
p.join()
45+
46+
if __name__ == '__main__':
47+
work(1)
48+
work(2)
49+
work(4)
50+
work(8)
51+
work2(1)
52+
work2(2)
53+
work2(4)
54+
work2(8)
55+
56+
# 测试结果：
57+
# 方法work开始时间：Thu Apr 30 00:26:21 2015
58+
# 方法work结束时间：Thu Apr 30 00:26:46 2015
59+
# 方法work运行时间：25.28S
60+
# 方法work开始时间：Thu Apr 30 00:26:46 2015
61+
# 方法work结束时间：Thu Apr 30 00:27:04 2015
62+
# 方法work运行时间：18.25S
63+
# 方法work开始时间：Thu Apr 30 00:27:04 2015
64+
# 方法work结束时间：Thu Apr 30 00:27:10 2015
65+
# 方法work运行时间：5.47S
66+
# 方法work开始时间：Thu Apr 30 00:27:10 2015
67+
# 方法work结束时间：Thu Apr 30 00:27:15 2015
68+
# 方法work运行时间：5.16S
69+
# 方法work2开始时间：Thu Apr 30 00:33:56 2015
70+
# 方法work2结束时间：Thu Apr 30 00:34:24 2015
71+
# 方法work2运行时间：27.42S
72+
# 方法work2开始时间：Thu Apr 30 00:34:24 2015
73+
# 方法work2结束时间：Thu Apr 30 00:34:35 2015
74+
# 方法work2运行时间：11.62S
75+
# 方法work2开始时间：Thu Apr 30 00:34:35 2015
76+
# 方法work2结束时间：Thu Apr 30 00:34:42 2015
77+
# 方法work2运行时间：6.87S
78+
# 方法work2开始时间：Thu Apr 30 00:34:42 2015
79+
# 方法work2结束时间：Thu Apr 30 00:34:53 2015
80+
# 方法work2运行时间：11.10S

‎performance_testing_2.py

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# encoding: utf-8
2+
__author__ = 'zhanghe'
3+
4+
import os
5+
from PIL import Image
6+
7+
SIZE = (75, 75)
8+
SAVE_DIRECTORY = 'thumbs'
9+
10+
11+
def get_image_paths(folder):
12+
return (os.path.join(folder, f)
13+
for f in os.listdir(folder)
14+
if 'jpeg' in f)
15+
16+
17+
def create_thumbnail(filename):
18+
im = Image.open(filename)
19+
im.thumbnail(SIZE, Image.ANTIALIAS)
20+
base, fname = os.path.split(filename)
21+
save_path = os.path.join(base, SAVE_DIRECTORY, fname)
22+
im.save(save_path)
23+
24+
25+
if __name__ == '__main__':
26+
img_folder = os.path.abspath(
27+
'11_18_2013_R000_IQM_Big_Sur_Mon__e10d1958e7b766c3e840')
28+
os.mkdir(os.path.join(img_folder, SAVE_DIRECTORY))
29+
30+
images = get_image_paths(img_folder)
31+
32+
for image in images:
33+
create_thumbnail(Image)

‎performance_testing_3.py

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# encoding: utf-8
2+
__author__ = 'zhanghe'
3+
4+
import os
5+
from multiprocessing import Pool
6+
from PIL import Image
7+
8+
SIZE = (75, 75)
9+
SAVE_DIRECTORY = 'thumbs'
10+
11+
12+
def get_image_paths(folder):
13+
return (os.path.join(folder, f)
14+
for f in os.listdir(folder)
15+
if 'jpeg' in f)
16+
17+
18+
def create_thumbnail(filename):
19+
im = Image.open(filename)
20+
im.thumbnail(SIZE, Image.ANTIALIAS)
21+
base, fname = os.path.split(filename)
22+
save_path = os.path.join(base, SAVE_DIRECTORY, fname)
23+
im.save(save_path)
24+
25+
26+
if __name__ == '__main__':
27+
img_folder = os.path.abspath(
28+
'11_18_2013_R000_IQM_Big_Sur_Mon__e10d1958e7b766c3e840')
29+
os.mkdir(os.path.join(img_folder, SAVE_DIRECTORY))
30+
31+
images = get_image_paths(img_folder)
32+
33+
pool = Pool()
34+
pool.map(create_thumbnail, images)
35+
pool.close()
36+
pool.join()

‎tools/time_log.pyc

1.99 KB
Binary file not shown.

0 commit comments

Comments
 (0)