Add httpclient module, using custom httphandler to do POST request

httpclient.py

@@ -0,0 +1,127 @@
+import sys, os
+import socket
+import urllib, urllib2, cookielib
+
+from urllib2_file import new_httphandler
+
+tmp = os.path.join(os.path.dirname(__file__), 'tests')
+sys.path.insert(0, tmp)
+import testHTTPServer
+
+class HTTPClient(object):
+    default_encode = "utf-8"
+    def __init__(self, cookie = False, encode = "utf-8"):
+        if cookie:
+            self.cookie = cookielib.MozillaCookieJar(str(cookie))
+        else:
+            self.cookie = False
+        self.request = None
+        self.encode = encode
+
+        ## Whether to use custome opener to post multipart/form-data
+        self.use_new_httphandler = False  
+        self.extra_handlers = []
+
+    def get(self, url, headers=False):
+        self.request = urllib2.Request(url)
+        self._before(headers, self.cookie);
+        resp = urllib2.urlopen(self.request)
+        return self._after(resp)
+
+    # Generally, post data is string or tuple 
+    # post file if data is dict
+    # example:data = {'form_name':{ 'fd':open('/lib/libresolv.so.2','filename':'libresolv.so'}}            
+    def post(self, url, data, headers={}):
+        if isinstance(data, dict):
+            self.use_new_httphandler = True
+
+        self._before(headers=headers, cookie=self.cookie);
+
+        if self.use_new_httphandler:
+            self.extra_handlers.append(new_httphandler)
+        else:
+            pass
+        self.request = urllib2.Request(url, headers=headers)
+        opener = urllib2.build_opener(*self.extra_handlers)    
+        resp = opener.open(self.request, data)
+
+        self.use_new_httphandler = False
+        self.extra_handlers = []
+
+        return self._after(resp)
+
+    def _before(self, headers = False, cookie = False):
+        if cookie != False:
+            self._handle_cookie(cookie)
+        if headers != False:
+            self._handle_header(headers)
+
+    def _after(self, resp):
+        if self.cookie:
+            self.cookie.save(ignore_discard = True, ignore_expires = True)
+
+        res = resp.read()
+        if self.encode != HTTPClient.default_encode:
+            return res.decode(self.encode, HTTPClient.default_encode)
+        return res
+
+    def _handle_cookie(self, cookie):
+        self.extra_handlers.append(urllib2.HTTPCookieProcessor(cookie))
+
+    def _handle_header(self, headers):
+        for (k,v) in headers.items():
+            self.request.add_header(k, v)
+
+
+if __name__ == '__main__':
+    # start http server
+    listen_port_start = 32800
+    for listen_port in range(listen_port_start, listen_port_start + 10):
+        # print "trying to bind on port %s" % listen_port
+        httpd = testHTTPServer.testHTTPServer('127.0.0.1', listen_port, )
+        try:
+            httpd.listen()
+            break
+        except socket.error, (errno, strerr):
+            # already in use
+            if errno == 98:
+                continue
+            else:
+                print "ERROR: listen: ", errno, strerr 
+                sys.exit(-1)
+    print "http server bound to port", listen_port
+    httpd.start()
+
+    httpserver = 'http://127.0.0.1:%s' % listen_port
+    while not httpd.isReady():
+        time.sleep(0.1)
+    u = urllib2.urlopen(httpserver + '/ping') 
+    data = u.read()
+    if data != "pong":
+        print "error"
+        sys.exit(-1)
+
+    client = HTTPClient(cookie=True)
+    post_data1 = {'file1': {'filename': 'file1_name',
+                           'fd': open('/bin/ls')}
+                 }
+    ret = client.post(httpserver, post_data1)
+    print ret
+
+    post_data2 = 'Text to be posted'
+    ret = client.post(httpserver, post_data2,
+                      headers={'Content-Type' : 'binary'})
+    ret = client.post(httpserver, post_data2,
+                      headers={'Content-Type' : 'text/html'})
+    print ret
+
+    post_data3 = 'var1=value1;var2=value2'
+    ret = client.post(httpserver, post_data3)
+    print ret
+
+    httpd.die()
+    httpd.join()
+    print "http server stopped"
+
+
+

tests/testHTTPServer.py

@@ -34,10 +34,18 @@ def do_GET(self):
     def do_POST(self):
         if not self.headers.has_key("Content-Type"):
             self.send_response(500)
-            self.wfile.write("needs content type\r\n")
             self.end_headers()
+            self.wfile.write("Needs Content-Type\n")
             return
-
+
+        if self.headers['Content-Type'].find('form') == -1:
+            self.send_response(200)
+            self.end_headers()
+            varLen = int(self.headers['Content-Length'])
+            postVars = self.rfile.read(varLen)
+            self.wfile.write('POST_%s length=%s\n' % (self.headers['Content-Type'], varLen))
+            return
+
         form = cgi.FieldStorage(fp = self.rfile,
                                 headers = self.headers,
                                 environ = {'REQUEST_METHOD':    'POST',
@@ -46,7 +54,7 @@ def do_POST(self):
         if len(form) == 0:
             self.send_response(200)
             self.end_headers()
-            self.wfile.write("NO_FORM_DATA found")
+            self.wfile.write("NO_FORM_DATA found\n")
             return
 
         self.send_response(200)
@@ -81,10 +89,27 @@ def die(self):
     def isReady(self):
         return self.is_ready
 
+httpd = None
 def main():
+    global httpd
+
+    import signal
+    import traceback
+    def receive_signal(signum, stack):
+        print '[main] Received signal:', signum
+        print '[main] Current stack:'
+        traceback.print_stack(stack)
+
+        httpd.die()
+        httpd.join()
+        sys.exit(0)
+
+
+    signal.signal(signal.SIGINT, receive_signal)
+
     listen_port_start = 32800
     for listen_port in range(listen_port_start, listen_port_start + 2):
-        print "trying to bind on port %s" % listen_port
+        print "[main] trying to bind on port %s" % listen_port
         httpd = testHTTPServer('127.0.0.1', listen_port, )
         try:
             httpd.listen()
@@ -99,10 +124,12 @@ def main():
                 sys.exit(-1)
 
     httpd.start()
-    print "started on port", listen_port
+    print "[main] started on port", listen_port
     try:
-        time.sleep(30)
+        time.sleep(9999999.)
+        print '[main] Time is up'
     except KeyboardInterrupt:
+        print '[main] Keyboard Interrupt'
         pass
     httpd.die()
     httpd.join()

urllib2_file.py

@@ -202,7 +202,15 @@ def send_data(v_vars, v_files, boundary, sock=None):
     return buffer_len
 
 # mainly a copy of HTTPHandler from urllib2
-class newHTTPHandler(urllib2.BaseHandler):
+class newHTTPHandler(urllib2.HTTPHandler):
+    """
+    Custom http handler to handle with the multipart/form-data POST request
+    """
+
+    def http_request(self, request):
+        ## The new http handler not use the builtin AbstractHTTPHandler._do_request in urlib2
+        return request
+
     def http_open(self, req):
         return self.do_open(httplib.HTTP, req)
 
@@ -284,7 +292,7 @@ def do_open(self, http_class, req):
                 data = urllib.urlencode(v_vars)
                 h.send(data)
             else:
-                # "normal" urllib2.urlopen()
+                # Normal case, single data (mostly string)
                 h.send(data)
 
         code, msg, hdrs = h.getreply()
@@ -297,12 +305,13 @@ def do_open(self, http_class, req):
         else:
             return self.parent.error('http', req, fp, code, msg, hdrs)
 
-urllib2._old_HTTPHandler = urllib2.HTTPHandler
-urllib2.HTTPHandler = newHTTPHandler
-
 class newHTTPSHandler(newHTTPHandler):
     def https_open(self, req):
         return self.do_open(httplib.HTTPS, req)
 
-urllib2.HTTPSHandler = newHTTPSHandler
+
+## Expose the handler for user to use urllib2.build_opener to generate a temporary opener,
+# print dir(urllib2.HTTPHandler)
+# print dir(newHTTPHandler)
+new_httphandler = newHTTPHandler