From 7001b7d972d7036d3a8adefad65d40ec13aa451e Mon Sep 17 00:00:00 2001 From: bruce Date: Thu, 5 Sep 2013 19:44:16 +0800 Subject: [PATCH] Add httpclient module, using custom httphandler to do POST request --- httpclient.py | 127 ++++++++++++++++++++++++++++++++++++++++ tests/testHTTPServer.py | 39 ++++++++++-- urllib2_file.py | 21 +++++-- 3 files changed, 175 insertions(+), 12 deletions(-) create mode 100644 httpclient.py diff --git a/httpclient.py b/httpclient.py new file mode 100644 index 0000000..c868e2c --- /dev/null +++ b/httpclient.py @@ -0,0 +1,127 @@ +import sys, os +import socket +import urllib, urllib2, cookielib + +from urllib2_file import new_httphandler + +tmp = os.path.join(os.path.dirname(__file__), 'tests') +sys.path.insert(0, tmp) +import testHTTPServer + +class HTTPClient(object): + default_encode = "utf-8" + def __init__(self, cookie = False, encode = "utf-8"): + if cookie: + self.cookie = cookielib.MozillaCookieJar(str(cookie)) + else: + self.cookie = False + self.request = None + self.encode = encode + + ## Whether to use custome opener to post multipart/form-data + self.use_new_httphandler = False + self.extra_handlers = [] + + def get(self, url, headers=False): + self.request = urllib2.Request(url) + self._before(headers, self.cookie); + resp = urllib2.urlopen(self.request) + return self._after(resp) + + # Generally, post data is string or tuple + # post file if data is dict + # example:data = {'form_name':{ 'fd':open('/lib/libresolv.so.2','filename':'libresolv.so'}} + def post(self, url, data, headers={}): + if isinstance(data, dict): + self.use_new_httphandler = True + + self._before(headers=headers, cookie=self.cookie); + + if self.use_new_httphandler: + self.extra_handlers.append(new_httphandler) + else: + pass + self.request = urllib2.Request(url, headers=headers) + opener = urllib2.build_opener(*self.extra_handlers) + resp = opener.open(self.request, data) + + self.use_new_httphandler = False + self.extra_handlers = [] + + return self._after(resp) + + def _before(self, headers = False, cookie = False): + if cookie != False: + self._handle_cookie(cookie) + if headers != False: + self._handle_header(headers) + + def _after(self, resp): + if self.cookie: + self.cookie.save(ignore_discard = True, ignore_expires = True) + + res = resp.read() + if self.encode != HTTPClient.default_encode: + return res.decode(self.encode, HTTPClient.default_encode) + return res + + def _handle_cookie(self, cookie): + self.extra_handlers.append(urllib2.HTTPCookieProcessor(cookie)) + + def _handle_header(self, headers): + for (k,v) in headers.items(): + self.request.add_header(k, v) + + +if __name__ == '__main__': + # start http server + listen_port_start = 32800 + for listen_port in range(listen_port_start, listen_port_start + 10): + # print "trying to bind on port %s" % listen_port + httpd = testHTTPServer.testHTTPServer('127.0.0.1', listen_port, ) + try: + httpd.listen() + break + except socket.error, (errno, strerr): + # already in use + if errno == 98: + continue + else: + print "ERROR: listen: ", errno, strerr + sys.exit(-1) + print "http server bound to port", listen_port + httpd.start() + + httpserver = 'http://127.0.0.1:%s' % listen_port + while not httpd.isReady(): + time.sleep(0.1) + u = urllib2.urlopen(httpserver + '/ping') + data = u.read() + if data != "pong": + print "error" + sys.exit(-1) + + client = HTTPClient(cookie=True) + post_data1 = {'file1': {'filename': 'file1_name', + 'fd': open('/bin/ls')} + } + ret = client.post(httpserver, post_data1) + print ret + + post_data2 = 'Text to be posted' + ret = client.post(httpserver, post_data2, + headers={'Content-Type' : 'binary'}) + ret = client.post(httpserver, post_data2, + headers={'Content-Type' : 'text/html'}) + print ret + + post_data3 = 'var1=value1;var2=value2' + ret = client.post(httpserver, post_data3) + print ret + + httpd.die() + httpd.join() + print "http server stopped" + + + diff --git a/tests/testHTTPServer.py b/tests/testHTTPServer.py index 590a503..583d60b 100755 --- a/tests/testHTTPServer.py +++ b/tests/testHTTPServer.py @@ -34,10 +34,18 @@ def do_GET(self): def do_POST(self): if not self.headers.has_key("Content-Type"): self.send_response(500) - self.wfile.write("needs content type\r\n") self.end_headers() + self.wfile.write("Needs Content-Type\n") return - + + if self.headers['Content-Type'].find('form') == -1: + self.send_response(200) + self.end_headers() + varLen = int(self.headers['Content-Length']) + postVars = self.rfile.read(varLen) + self.wfile.write('POST_%s length=%s\n' % (self.headers['Content-Type'], varLen)) + return + form = cgi.FieldStorage(fp = self.rfile, headers = self.headers, environ = {'REQUEST_METHOD': 'POST', @@ -46,7 +54,7 @@ def do_POST(self): if len(form) == 0: self.send_response(200) self.end_headers() - self.wfile.write("NO_FORM_DATA found") + self.wfile.write("NO_FORM_DATA found\n") return self.send_response(200) @@ -81,10 +89,27 @@ def die(self): def isReady(self): return self.is_ready +httpd = None def main(): + global httpd + + import signal + import traceback + def receive_signal(signum, stack): + print '[main] Received signal:', signum + print '[main] Current stack:' + traceback.print_stack(stack) + + httpd.die() + httpd.join() + sys.exit(0) + + + signal.signal(signal.SIGINT, receive_signal) + listen_port_start = 32800 for listen_port in range(listen_port_start, listen_port_start + 2): - print "trying to bind on port %s" % listen_port + print "[main] trying to bind on port %s" % listen_port httpd = testHTTPServer('127.0.0.1', listen_port, ) try: httpd.listen() @@ -99,10 +124,12 @@ def main(): sys.exit(-1) httpd.start() - print "started on port", listen_port + print "[main] started on port", listen_port try: - time.sleep(30) + time.sleep(9999999.) + print '[main] Time is up' except KeyboardInterrupt: + print '[main] Keyboard Interrupt' pass httpd.die() httpd.join() diff --git a/urllib2_file.py b/urllib2_file.py index 6af9f98..1f47e73 100755 --- a/urllib2_file.py +++ b/urllib2_file.py @@ -202,7 +202,15 @@ def send_data(v_vars, v_files, boundary, sock=None): return buffer_len # mainly a copy of HTTPHandler from urllib2 -class newHTTPHandler(urllib2.BaseHandler): +class newHTTPHandler(urllib2.HTTPHandler): + """ + Custom http handler to handle with the multipart/form-data POST request + """ + + def http_request(self, request): + ## The new http handler not use the builtin AbstractHTTPHandler._do_request in urlib2 + return request + def http_open(self, req): return self.do_open(httplib.HTTP, req) @@ -284,7 +292,7 @@ def do_open(self, http_class, req): data = urllib.urlencode(v_vars) h.send(data) else: - # "normal" urllib2.urlopen() + # Normal case, single data (mostly string) h.send(data) code, msg, hdrs = h.getreply() @@ -297,12 +305,13 @@ def do_open(self, http_class, req): else: return self.parent.error('http', req, fp, code, msg, hdrs) -urllib2._old_HTTPHandler = urllib2.HTTPHandler -urllib2.HTTPHandler = newHTTPHandler - class newHTTPSHandler(newHTTPHandler): def https_open(self, req): return self.do_open(httplib.HTTPS, req) -urllib2.HTTPSHandler = newHTTPSHandler + +## Expose the handler for user to use urllib2.build_opener to generate a temporary opener, +# print dir(urllib2.HTTPHandler) +# print dir(newHTTPHandler) +new_httphandler = newHTTPHandler