Merge lp:~mvo/piston-mini-client/proxy_type_http_no_tunnel into lp:piston-mini-client

[Michael Vogt]

This adds support for a new type of http proxy that is sufficient for plain http tunneling to the internals version of socks.py. It will also patch httplib2 to use the internal version of socks.py over the one bundled with httplib2 that has no support for this. See http://code.google.com/p/httplib2/issues/detail?id=38 for some more details

[lp:~mvo/piston-mini-client/proxy_type_http_no_tunnel]

53. By Michael Vogt on 2011-10-04

piston_mini_client/__init__.py: always import httplib2 first

54. By Michael Vogt on 2011-10-04

use logging.warn() instead of print

[Anthony Lenton (elachuni)]

Thanks for the fix mvo!

review: Approve

=== modified file 'piston_mini_client/__init__.py'
--- piston_mini_client/__init__.py	2011-09-28 08:32:48 +0000
+++ piston_mini_client/__init__.py	2011-10-04 13:34:17 +0000
@@ -3,6 +3,17 @@
 # GNU Lesser General Public License version 3 (see the file LICENSE).
 
 import httplib2
+
+try:
+    # ensure we have a version with a fix for 
+    #  http://code.google.com/p/httplib2/issues/detail?id=38
+    # and if not, patch in our own socks with the fix
+    from httplib2.socks import PROXY_TYPE_HTTP_NO_TUNNEL
+    from httplib2 import socks
+except ImportError:
+    from piston_mini_client import socks            
+    httplib2.socks = socks
+
 import json
 import os
 import urllib
@@ -262,19 +273,19 @@
     def _get_proxy_info(self, scheme):
         envvar = "%s_proxy" % scheme
         if envvar in os.environ:
-            try:
-                # In Oneiric socks is already packaged as part of httplib2.
-                from httplib2 import socks
-            except ImportError:
-                from piston_mini_client import socks            
             url = urlparse(os.environ[envvar])
             user_pass, sep, host_and_port = url.netloc.rpartition("@")
             user, sep, passw = user_pass.partition(":")
             host, sep, port = host_and_port.partition(":")
             if port:
                 port = int(port)
+            proxy_type = socks.PROXY_TYPE_HTTP
+            if scheme == "http":
+                # this will not require the CONNECT acl from squid and
+                # is good enough for http connections
+                proxy_type = socks.PROXY_TYPE_HTTP_NO_TUNNEL
             proxy_info = httplib2.ProxyInfo(
-                proxy_type=socks.PROXY_TYPE_HTTP,
+                proxy_type=proxy_type,
                 proxy_host=host,
                 proxy_port=port or 8080,
                 proxy_user=user or None,

=== modified file 'piston_mini_client/socks.py'
--- piston_mini_client/socks.py	2011-09-27 19:28:32 +0000
+++ piston_mini_client/socks.py	2011-10-04 13:34:17 +0000
@@ -43,13 +43,13 @@
 import socket
 import struct
 import sys
-
-if getattr(socket, 'socket', None) is None:
-    raise ImportError('socket.socket missing, proxy support unusable')
+import base64
+import logging
 
 PROXY_TYPE_SOCKS4 = 1
 PROXY_TYPE_SOCKS5 = 2
 PROXY_TYPE_HTTP = 3
+PROXY_TYPE_HTTP_NO_TUNNEL = 4
 
 _defaultproxy = None
 _orgsocket = socket.socket
@@ -127,6 +127,8 @@
         self.__proxysockname = None
         self.__proxypeername = None
 
+        self.__httptunnel = True
+
     def __recvall(self, count):
         """__recvall(count) -> data
         Receive EXACTLY the number of bytes requested from the socket.
@@ -139,6 +141,43 @@
             data = data + d
         return data
 
+    def sendall(self, content, *args):
+        """ override socket.socket.sendall method to rewrite the header 
+        for non-tunneling proxies if needed 
+        """
+        if not self.__httptunnel:
+            content = self.__rewriteproxy(content)
+
+        return super(socksocket, self).sendall(content, *args)
+
+    def __rewriteproxy(self, header):
+        """ rewrite HTTP request headers to support non-tunneling proxies 
+        (i.e. thos which do not support the CONNECT method).
+        This only works for HTTP (not HTTPS) since HTTPS requires tunneling.
+        """
+        host, endpt = None, None
+        hdrs = header.split("\r\n")
+        for hdr in hdrs:
+            if hdr.lower().startswith("host:"):
+                host = hdr
+            elif hdr.lower().startswith("get") or hdr.lower().startswith("post"):
+                endpt = hdr
+        if host and endpt: 
+            hdrs.remove(host)
+            hdrs.remove(endpt)
+            host = host.split(" ")[1]
+            endpt = endpt.split(" ")
+            if (self.__proxy[4] != None and self.__proxy[5] != None):
+                hdrs.insert(0, self.__getauthheader())
+            hdrs.insert(0, "Host: %s" % host)
+            hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2]))
+
+        return "\r\n".join(hdrs)
+
+    def __getauthheader(self):
+        auth = self.__proxy[4] + ":" + self.__proxy[5]
+        return "Proxy-Authorization: Basic " + base64.b64encode(auth)
+
     def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
         """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
         Sets the proxy to be used.
@@ -326,7 +365,12 @@
             addr = socket.gethostbyname(destaddr)
         else:
             addr = destaddr
-        self.sendall(("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n").encode())
+        headers =  "CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n"
+        headers += "Host: " + destaddr + "\r\n"
+        if (self.__proxy[4] != None and self.__proxy[5] != None):
+                headers += self.__getauthheader() + "\r\n"
+        headers += "\r\n"
+        self.sendall(headers.encode())
         # We read the response until we get the string "\r\n\r\n"
         resp = self.recv(1)
         while resp.find("\r\n\r\n".encode()) == -1:
@@ -379,6 +423,17 @@
                 portnum = 8080
             _orgsocket.connect(self,(self.__proxy[1], portnum))
             self.__negotiatehttp(destpair[0], destpair[1])
+        elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL:
+            if self.__proxy[2] != None:
+                portnum = self.__proxy[2]
+            else:
+                portnum = 8080
+            _orgsocket.connect(self,(self.__proxy[1],portnum))
+            if destpair[1] == 443:
+                logging.warn("SSL connections (generally on port 443) require the use of tunneling - failing back to PROXY_TYPE_HTTP")
+                self.__negotiatehttp(destpair[0],destpair[1])
+            else:
+                self.__httptunnel = False
         elif self.__proxy[0] == None:
             _orgsocket.connect(self, (destpair[0], destpair[1]))
         else:

=== modified file 'piston_mini_client/tests/test_proxy.py'
--- piston_mini_client/tests/test_proxy.py	2011-09-27 16:09:08 +0000
+++ piston_mini_client/tests/test_proxy.py	2011-10-04 13:34:17 +0000
@@ -53,6 +53,12 @@
         proxy_info = api._http["http"].proxy_info
         self.assertEqual(proxy_info, None)
 
+    def test_httplib2_proxy_type_http_no_tunnel(self):
+        # test that patching httplib2 with our own socks.py works
+        import httplib2
+        import piston_mini_client
+        self.assertEqual(piston_mini_client.socks.PROXY_TYPE_HTTP_NO_TUNNEL,
+                         httplib2.socks.PROXY_TYPE_HTTP_NO_TUNNEL)
 
 if __name__ == "__main__":
     import unittest