Merge lp:~dobey/ubuntuone-dev-tools/updates-2992 into lp:ubuntuone-dev-tools/stable-3-0

[dobey]

[Manuel de la Peña]

    Provide batch scripts for u1trial and u1lint on Windows.
    Provide a new SquidTestCase for running tests with a private squid instance.

[Rodney Dawes]

    Add missing squid.conf.in to setup.py

[Diego Sarmentero (diegosarmentero)]

+1

review: Approve

[Brian Curtin (brian.curtin)]

Minor issue: In your uses of subprocess.Popen, you open stdout and stderr but don't close them.

Otherwise: +1

review: Approve

[dobey (dobey)]

review: Abstain

=== added file 'bin/u1lint.bat'
--- bin/u1lint.bat	1970-01-01 00:00:00 +0000
+++ bin/u1lint.bat	2012-01-17 21:43:26 +0000
@@ -0,0 +1,17 @@
+::
+:: Copyright 2012 Canonical Ltd.
+::
+:: This program is free software: you can redistribute it and/or modify it
+:: under the terms of the GNU General Public License version 3, as published
+:: by the Free Software Foundation.
+::
+:: This program is distributed in the hope that it will be useful, but
+:: WITHOUT ANY WARRANTY; without even the implied warranties of
+:: MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+:: PURPOSE.  See the GNU General Public License for more details.
+::
+:: You should have received a copy of the GNU General Public License along
+:: with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+:: Use python to execute the script having the same name as this batch
+python "%~dpn0" %*

=== added file 'bin/u1trial.bat'
--- bin/u1trial.bat	1970-01-01 00:00:00 +0000
+++ bin/u1trial.bat	2012-01-17 21:43:26 +0000
@@ -0,0 +1,17 @@
+::
+:: Copyright 2012 Canonical Ltd.
+::
+:: This program is free software: you can redistribute it and/or modify it
+:: under the terms of the GNU General Public License version 3, as published
+:: by the Free Software Foundation.
+::
+:: This program is distributed in the hope that it will be useful, but
+:: WITHOUT ANY WARRANTY; without even the implied warranties of
+:: MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+:: PURPOSE.  See the GNU General Public License for more details.
+::
+:: You should have received a copy of the GNU General Public License along
+:: with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+:: Use python to execute the script having the same name as this batch
+python "%~dpn0" %*

=== added file 'data/squid.conf.in'
--- data/squid.conf.in	1970-01-01 00:00:00 +0000
+++ data/squid.conf.in	2012-01-17 21:43:26 +0000
@@ -0,0 +1,120 @@
+auth_param basic casesensitive on
+# Use a default auth using ncsa and the passed generated file.
+auth_param basic program ${auth_process} ${auth_file}
+#Recommended minimum configuration:
+acl manager proto cache_object
+acl localhost src 127.0.0.1/32
+acl to_localhost dst 127.0.0.0/32
+#
+# Example rule allowing access from your local networks.
+# Adapt to list your (internal) IP networks from where browsing
+# should be allowed
+acl all src all
+acl localnet src 10.0.0.0/8	# RFC1918 possible internal network
+acl localnet src 172.16.0.0/12	# RFC1918 possible internal network
+acl localnet src 192.168.0.0/16	# RFC1918 possible internal network
+#
+acl SSL_ports port 443		# https
+acl SSL_ports port 563		# snews
+acl SSL_ports port 873		# rsync
+acl Safe_ports port 80		# http
+acl Safe_ports port 21		# ftp
+acl Safe_ports port 443		# https
+acl Safe_ports port 70		# gopher
+acl Safe_ports port 210		# wais
+acl Safe_ports port 1025-65535	# unregistered ports
+acl Safe_ports port 280		# http-mgmt
+acl Safe_ports port 488		# gss-http
+acl Safe_ports port 591		# filemaker
+acl Safe_ports port 777		# multiling http
+acl Safe_ports port 631		# cups
+acl Safe_ports port 873		# rsync
+acl Safe_ports port 901		# SWAT
+acl purge method PURGE
+acl CONNECT method CONNECT
+
+# make an acl for users that have auth
+acl password proxy_auth REQUIRED myportname ${auth_port_number}
+acl auth_port_connected myportname ${auth_port_number}
+acl nonauth_port_connected myportname ${noauth_port_number}
+
+# Settings used for the tests:
+# Allow users connected to the nonauth port
+# Allow users authenticated AND connected to the auth port
+http_access allow nonauth_port_connected
+http_access allow password
+
+#Recommended minimum configuration:
+#
+# Only allow cachemgr access from localhost
+http_access allow manager localhost
+http_access deny manager
+# Only allow purge requests from localhost
+http_access allow purge localhost
+http_access deny purge
+# Deny requests to unknown ports
+http_access deny !Safe_ports
+# Deny CONNECT to other than SSL ports
+http_access deny CONNECT !SSL_ports
+# Example rule allowing access from your local networks.
+# Adapt localnet in the ACL section to list your (internal) IP networks
+# from where browsing should be allowed
+#http_access allow localnet
+http_access allow localhost
+
+# And finally deny all other access to this proxy
+http_access deny all
+
+icp_access allow localnet
+icp_access deny all
+
+# Squid normally listens to port 3128 but we are going to listento two
+# different ports, one for auth one for nonauth.
+http_port ${noauth_port_number}
+http_port ${auth_port_number}
+
+#We recommend you to use at least the following line.
+hierarchy_stoplist cgi-bin ?
+
+# Default cache settings.
+cache_dir ufs ${spool_temp} 1000 16 256
+
+# access log settings
+access_log ${squid_temp}/access.log squid
+
+# cache log settings
+cache_log ${squid_temp}/cache.log
+
+# cache store log settings
+cache_store_log ${squid_temp}/store.log
+
+# mime table conf
+# mime_table /usr/share/squid/mime.conf
+
+#Default pid file name
+pid_filename ${squid_temp}/squid.pid
+
+# debug options (Full debugging)
+debug_options ALL,1
+
+#Default netdb_filename
+
+#Suggested default:
+refresh_pattern ^ftp:		1440	20%	10080
+refresh_pattern ^gopher:	1440	0%	1440
+refresh_pattern -i (/cgi-bin/|\?) 0	0%	0
+refresh_pattern (Release|Packages(.gz)*)$	0	20%	2880
+# example line deb packages
+refresh_pattern .		0	20%	4320
+
+# Don't upgrade ShoutCast responses to HTTP
+acl shoutcast rep_header X-HTTP09-First-Line ^ICY.[0-9]
+
+# Apache mod_gzip and mod_deflate known to be broken so don't trust
+# Apache to signal ETag correctly on such responses
+acl apache rep_header Server ^Apache
+
+hosts_file /etc/hosts
+
+# Leave coredumps in the first cache dir
+coredump_dir ${spool_temp}/squid

=== modified file 'run-tests'
--- run-tests	2011-11-21 19:58:40 +0000
+++ run-tests	2012-01-17 21:43:26 +0000
@@ -20,6 +20,6 @@
 bin/u1trial --reactor=twisted ubuntuone
 echo "Running style checks..."
 bin/u1lint
-pep8 --repeat . bin/*
+pep8 --repeat . bin/* --exclude=*.bat
 rm -rf _trial_temp
 rm -rf .coverage

=== modified file 'setup.py'
--- setup.py	2012-01-03 20:29:17 +0000
+++ setup.py	2012-01-17 21:43:26 +0000
@@ -44,6 +44,15 @@
         if retcode != 0:
             sys.exit(retcode)
 
+# pylint: disable=C0103
+scripts = ['bin/u1lint',
+           'bin/u1trial']
+# pylint: enable=C0103
+
+if sys.platform == 'win32':
+    # lets add the .bat so that windows users are happy
+    scripts.extend(['bin/u1lint.bat', 'bin/u1trial.bat'])
+
 setup(name=PACKAGE,
       version=VERSION,
       description='Ubuntu One development tools and utilities',
@@ -55,12 +64,11 @@
                 'ubuntuone.devtools.testing',
                 'ubuntuone.devtools.testcases'],
       extra_path='ubuntuone-dev-tools',
-      scripts=['bin/u1lint',
-               'bin/u1trial',
-               ],
+      scripts=scripts,
       data_files=[('share/%s' % PACKAGE,
                    ['pylintrc',
-                    'data/dbus-session.conf.in']),
+                    'data/dbus-session.conf.in',
+                    'data/squid.conf.in']),
                   ('share/man/man1',
                    ['man/u1lint.1',
                     'man/u1trial.1']),

=== modified file 'ubuntuone/devtools/services/__init__.py'
--- ubuntuone/devtools/services/__init__.py	2010-11-03 21:04:06 +0000
+++ ubuntuone/devtools/services/__init__.py	2012-01-17 21:43:26 +0000
@@ -1,1 +1,54 @@
+#
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Service runners for testing."""
+
+import os
+import socket
+
+from dirspec.basedir import load_data_paths
+
+
+def find_config_file(in_config_file):
+    """Find the first appropriate conf to use."""
+    # In case we're running from within the source tree
+    path = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                            os.path.pardir, os.path.pardir,
+                                            os.path.pardir,
+                                            "data", in_config_file))
+    if not os.path.exists(path):
+        # Use the installed file in $pkgdatadir as source
+        for path in load_data_paths("ubuntuone-dev-tools",
+                                    in_config_file):
+            if os.path.exists(path):
+                break
+
+    # Check to make sure we didn't just fall out of the loop
+    if not os.path.exists(path):
+        raise IOError('Could not locate suitable %s' % in_config_file)
+    return path
+
+
+def get_arbitrary_port():
+    """
+    Find an unused port, and return it.
+
+    There might be a small race condition here, but we aren't
+    worried about it.
+    """
+    sock = socket.socket()
+    sock.bind(('localhost', 0))
+    _, port = sock.getsockname()
+    sock.close()
+    return port

=== modified file 'ubuntuone/devtools/services/dbus.py'
--- ubuntuone/devtools/services/dbus.py	2011-12-19 22:37:31 +0000
+++ ubuntuone/devtools/services/dbus.py	2012-01-17 21:43:26 +0000
@@ -20,10 +20,12 @@
 import signal
 import subprocess
 
-from dirspec.basedir import load_data_paths
 from distutils.spawn import find_executable
 from urllib import quote
 
+from ubuntuone.devtools.services import find_config_file
+DBUS_CONFIG_FILE = 'dbus-session.conf.in'
+
 
 class DBusLaunchError(Exception):
     """Error while launching dbus-daemon"""
@@ -44,42 +46,25 @@
         self.running = False
         self.config_file = None
 
-    def _find_config_file(self, tempdir=None):
+    def _generate_config_file(self, tempdir=None):
         """Find the first appropriate dbus-session.conf to use."""
-        # In case we're running from within the source tree
-        path = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                            os.path.pardir, os.path.pardir,
-                                            os.path.pardir,
-                                            "data", "dbus-session.conf.in"))
-        if not os.path.exists(path):
-            # Use the installed file in $pkgdatadir as source
-            for path in load_data_paths("ubuntuone-dev-tools",
-                                        "dbus-session.conf.in"):
-                if os.path.exists(path):
-                    break
-
-        # Check to make sure we didn't just fall out of the loop
-        if not os.path.exists(path):
-            raise IOError('Could not locate suitable dbus-session.conf.in')
-
+        # load the config file
+        path = find_config_file(DBUS_CONFIG_FILE)
+        # replace config settings
         self.config_file = os.path.join(tempdir, 'dbus-session.conf')
         dbus_address = 'unix:tmpdir=%s' % quote(tempdir)
         with open(path) as in_file:
             content = in_file.read()
             with open(self.config_file, 'w') as out_file:
                 out_file.write(content.replace('@ADDRESS@', dbus_address))
-                out_file.close()
-            in_file.close()
 
     def start_service(self, tempdir=None):
         """Start our own session bus daemon for testing."""
-        if not tempdir:
-            tempdir = os.path.join(os.getcwd(), '_trial_temp')
         dbus = find_executable("dbus-daemon")
         if not dbus:
             raise NotFoundError("dbus-daemon was not found.")
 
-        self._find_config_file(tempdir)
+        self._generate_config_file(tempdir)
 
         dbus_args = ["--fork",
                      "--config-file=" + self.config_file,

=== added file 'ubuntuone/devtools/services/squid.py'
--- ubuntuone/devtools/services/squid.py	1970-01-01 00:00:00 +0000
+++ ubuntuone/devtools/services/squid.py	2012-01-17 21:43:26 +0000
@@ -0,0 +1,245 @@
+#
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Utilities for finding and running a squid proxy for testing."""
+
+import random
+import signal
+# pylint:disable=W0402
+import string
+# pylint:enable=W0402
+import subprocess
+import time
+
+from json import dumps, loads
+from os import environ, makedirs, kill, unlink
+from os.path import abspath, exists, join
+
+from distutils.spawn import find_executable
+
+from ubuntuone.devtools.services import (
+    find_config_file,
+    get_arbitrary_port,
+)
+
+SQUID_CONFIG_FILE = 'squid.conf.in'
+SQUID_DIR = 'squid'
+AUTH_PROCESS_PATH = '/usr/lib/%s/ncsa_auth'
+SPOOL_DIR = 'spool'
+AUTH_FILE = 'htpasswd'
+PROXY_ENV_VAR = 'SQUID_PROXY_SETTINGS'
+
+
+def get_squid_executable():
+    """Return the squid executable of the system."""
+    # try with squid and if not present try with squid3 for newer systems
+    # (Ubuntu P). We also return the path to the auth process so that we can
+    # point to the correct one.
+    squid = find_executable('squid3')
+    auth_process = AUTH_PROCESS_PATH % 'squid3'
+    if squid is None:
+        squid = find_executable('squid')
+        auth_process = AUTH_PROCESS_PATH % 'squid'
+    return squid, auth_process
+
+
+def get_htpasswd_executable():
+    """Return the htpasswd executable."""
+    return find_executable('htpasswd')
+
+
+def _make_random_string(count):
+    """Make a random string of the given length."""
+    entropy = random.SystemRandom()
+    return ''.join([entropy.choice(string.letters) for _ in
+                   range(count)])
+
+
+def _get_basedir(tempdir):
+    """Return the base squid config."""
+    basedir = join(tempdir, SQUID_DIR)
+    basedir = abspath(basedir)
+    if not exists(basedir):
+        makedirs(basedir)
+    return basedir
+
+
+def _get_spool_temp_path(tempdir=''):
+    """Return the temp dir to be used for spool."""
+    basedir = _get_basedir(tempdir)
+    path = join(basedir, SPOOL_DIR)
+    path = abspath(path)
+    if not exists(path):
+        makedirs(path)
+    return path
+
+
+def _get_squid_temp_path(tempdir=''):
+    """Return the temp dir to be used by squid."""
+    basedir = _get_basedir(tempdir)
+    path = join(basedir, SQUID_DIR)
+    path = abspath(path)
+    if not exists(path):
+        makedirs(path)
+    return path
+
+
+def _get_auth_temp_path(tempdir=''):
+    """Return the path for the auth file."""
+    basedir = _get_basedir(tempdir)
+    auth_file = join(basedir, AUTH_FILE)
+    if not exists(basedir):
+        makedirs(basedir)
+    return auth_file
+
+
+def store_proxy_settings(settings):
+    """Store the proxy setting in an env var."""
+    environ[PROXY_ENV_VAR] = dumps(settings)
+
+
+def retrieve_proxy_settings():
+    """Return the proxy settings of the env."""
+    if PROXY_ENV_VAR in environ:
+        return loads(environ[PROXY_ENV_VAR])
+    return None
+
+
+def delete_proxy_settings():
+    """Delete the proxy env settings."""
+    if PROXY_ENV_VAR in environ:
+        del environ[PROXY_ENV_VAR]
+
+
+class SquidLaunchError(Exception):
+    """Error while launching squid."""
+
+
+class SquidRunner(object):
+    """Class for running a squid proxy with the local config."""
+
+    def __init__(self):
+        """Create a new instance."""
+        self.squid, self.auth_process = get_squid_executable()
+        if self.squid is None:
+            raise SquidLaunchError('Could not locate "squid".')
+
+        self.htpasswd = get_htpasswd_executable()
+        if self.htpasswd is None:
+            raise SquidLaunchError('Could not locate "htpasswd".')
+
+        self.settings = dict(noauth_port=None, auth_port=None,
+                             username=None, password=None)
+        self.squid_pid = None
+        self.running = False
+        self.config_file = None
+        self.auth_file = None
+
+    def _generate_config_file(self, tempdir=''):
+        """Find the first appropiate squid.conf to use."""
+        # load the config file
+        path = find_config_file(SQUID_CONFIG_FILE)
+        # replace config settings
+        basedir = join(tempdir, 'squid')
+        basedir = abspath(basedir)
+        if not exists(basedir):
+            makedirs(basedir)
+        self.config_file = join(basedir, 'squid.conf')
+        with open(path) as in_file:
+            template = string.Template(in_file.read())
+
+        self.settings['noauth_port'] = get_arbitrary_port()
+        self.settings['auth_port'] = get_arbitrary_port()
+        spool_path = _get_spool_temp_path(tempdir)
+        squid_path = _get_squid_temp_path(tempdir)
+        with open(self.config_file, 'w') as out_file:
+            out_file.write(template.safe_substitute(
+                              auth_file=self.auth_file,
+                              auth_process=self.auth_process,
+                              noauth_port_number=self.settings['noauth_port'],
+                              auth_port_number=self.settings['auth_port'],
+                              spool_temp=spool_path,
+                              squid_temp=squid_path))
+
+    def _generate_swap(self, config_file):
+        """Generate the squid swap files."""
+        squid_args = ['-z', '-f', config_file]
+        sp = subprocess.Popen([self.squid] + squid_args,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+        sp.wait()
+
+    def _generate_auth_file(self, tempdir=''):
+        """Generates a auth file using htpasswd."""
+        if self.settings['username'] is None:
+            self.settings['username'] = _make_random_string(10)
+        if self.settings['password'] is None:
+            self.settings['password'] = _make_random_string(10)
+
+        self.auth_file = _get_auth_temp_path(tempdir)
+        # remove possible old auth file
+        if exists(self.auth_file):
+            unlink(self.auth_file)
+        # create a new htpasswrd
+        htpasswd_args = ['-bc',
+                         self.auth_file,
+                         self.settings['username'],
+                         self.settings['password']]
+        sp = subprocess.Popen([self.htpasswd] + htpasswd_args,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+        sp.wait()
+
+    def _is_squid_running(self):
+        """Return if squid is running."""
+        squid_args = ['-k', 'check', '-f', self.config_file]
+        print 'Starting squid version...'
+        message = 'Waiting for squid to start...'
+        for timeout in (0.4, 0.1, 0.1, 0.2, 0.5, 1, 3, 5):
+            try:
+                #  Do not use stdout=PIPE or stderr=PIPE with this function.
+                subprocess.check_call([self.squid] + squid_args,
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE)
+                return True
+            except subprocess.CalledProcessError:
+                message += '.'
+                print message
+                time.sleep(timeout)
+        return False
+
+    def start_service(self, tempdir=None):
+        """Start our own proxy."""
+        # generate auth, config and swap dirs
+        self._generate_auth_file(tempdir)
+        self._generate_config_file(tempdir)
+        self._generate_swap(self.config_file)
+        squid_args = ['-N', '-X', '-f', self.config_file]
+        sp = subprocess.Popen([self.squid] + squid_args,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+        store_proxy_settings(self.settings)
+        if not self._is_squid_running():
+            raise SquidLaunchError('Could not start squid.')
+        self.squid_pid = sp.pid
+        self.running = True
+
+    def stop_service(self):
+        """Stop our proxy,"""
+        kill(self.squid_pid, signal.SIGKILL)
+        delete_proxy_settings()
+        self.running = False
+        unlink(self.config_file)
+        unlink(self.auth_file)
+        self.config_file = None

=== modified file 'ubuntuone/devtools/services/tests/test_dbus.py'
--- ubuntuone/devtools/services/tests/test_dbus.py	2011-11-23 11:31:08 +0000
+++ ubuntuone/devtools/services/tests/test_dbus.py	2012-01-17 21:43:26 +0000
@@ -25,6 +25,6 @@
         runner = DBusRunner()
         # pylint: disable=W0212
         os.makedirs(self.tmpdir)
-        runner._find_config_file(tempdir=self.tmpdir)
+        runner._generate_config_file(tempdir=self.tmpdir)
         shutil.rmtree(self.tmpdir)
         self.assertEqual(expected, runner.config_file)

=== added file 'ubuntuone/devtools/services/tests/test_squid.py'
--- ubuntuone/devtools/services/tests/test_squid.py	1970-01-01 00:00:00 +0000
+++ ubuntuone/devtools/services/tests/test_squid.py	2012-01-17 21:43:26 +0000
@@ -0,0 +1,334 @@
+#
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Test the squid service."""
+
+import json
+import os
+
+from twisted.internet import defer
+
+from ubuntuone.devtools.testcases import BaseTestCase
+from ubuntuone.devtools.services import squid
+
+# yes, we know names with _ are private
+#pylint: disable=W0212
+
+
+class PathsTestCase(BaseTestCase):
+    """Test the different path functions."""
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        """Set the different tests."""
+        yield super(PathsTestCase, self).setUp()
+        self.basedir_fn = squid._get_basedir
+        self.basedir = self.mktemp('paths')
+        self.path_exists = False
+        self.created_paths = []
+        self.called = []
+
+        def fake_basedir_fn(tempdir):
+            """Retun the base dir."""
+            self.called.append(('fake_basedir_fn', tempdir))
+            return self.basedir
+
+        def fake_makedirs(path):
+            """Fake the makedirs function."""
+            self.called.append(('fake_makedirs', path))
+            self.created_paths.append(path)
+
+        def fake_exists(path):
+            """Fake the exists method."""
+            self.called.append(('fake_exists', path))
+            return path in self.created_paths or self.path_exists
+
+        self.patch(squid, '_get_basedir', fake_basedir_fn)
+        self.patch(squid, 'makedirs', fake_makedirs)
+        self.patch(squid, 'exists', fake_exists)
+
+    def test_get_basedir_missing(self):
+        """Test the base dir creation."""
+        basedir = self.basedir_fn(self.basedir)
+        expected_path = os.path.join(self.basedir, squid.SQUID_DIR)
+        self.assertEqual(expected_path, basedir)
+        self.assertTrue(('fake_makedirs', expected_path) in self.called)
+        self.assertTrue(expected_path in self.created_paths)
+        self.assertTrue(('fake_exists', expected_path) in self.called)
+
+    def test_get_basedir_present(self):
+        """Test the base dir creation."""
+        self.path_exists = True
+        basedir = self.basedir_fn(self.basedir)
+        expected_path = os.path.join(self.basedir, squid.SQUID_DIR)
+        self.assertEqual(expected_path, basedir)
+        expected_path = os.path.join(self.basedir, squid.SQUID_DIR)
+        self.assertTrue(('fake_makedirs', expected_path) not in self.called)
+        self.assertTrue(expected_path not in self.created_paths)
+        self.assertTrue(('fake_exists', expected_path) in self.called)
+
+    def test_get_spool_temp_path_missing(self):
+        """Test the spool path creation."""
+        expected_path = os.path.join(self.basedir, squid.SPOOL_DIR)
+        result = squid._get_spool_temp_path()
+        self.assertEqual(expected_path, result)
+        self.assertTrue(('fake_basedir_fn', '') in self.called)
+        self.assertTrue(('fake_makedirs', expected_path) in self.called)
+        self.assertTrue(expected_path in self.created_paths)
+        self.assertTrue(('fake_exists', expected_path) in self.called)
+
+    def test_get_spool_temp_path_present(self):
+        """Test the spool path creation."""
+        self.path_exists = True
+        expected_path = os.path.join(self.basedir, squid.SPOOL_DIR)
+        result = squid._get_spool_temp_path()
+        self.assertEqual(expected_path, result)
+        self.assertTrue(('fake_basedir_fn', '') in self.called)
+        self.assertTrue(('fake_makedirs', expected_path) not in self.called)
+        self.assertTrue(expected_path not in self.created_paths)
+        self.assertTrue(('fake_exists', expected_path) in self.called)
+
+    def test_get_squid_temp_path_missing(self):
+        """Test the squid path creation."""
+        expected_path = os.path.join(self.basedir, squid.SQUID_DIR)
+        result = squid._get_squid_temp_path()
+        self.assertEqual(expected_path, result)
+        self.assertTrue(('fake_basedir_fn', '') in self.called)
+        self.assertTrue(('fake_makedirs', expected_path) in self.called)
+        self.assertTrue(expected_path in self.created_paths)
+        self.assertTrue(('fake_exists', expected_path) in self.called)
+
+    def test_get_squid_temp_path_present(self):
+        """Test the squid path creation."""
+        self.path_exists = True
+        expected_path = os.path.join(self.basedir, squid.SQUID_DIR)
+        result = squid._get_squid_temp_path()
+        self.assertEqual(expected_path, result)
+        self.assertTrue(('fake_basedir_fn', '') in self.called)
+        self.assertTrue(('fake_makedirs', expected_path) not in self.called)
+        self.assertTrue(expected_path not in self.created_paths)
+        self.assertTrue(('fake_exists', expected_path) in self.called)
+
+    def test_get_auth_temp_path(self):
+        """Test the creation of the auth path."""
+        self.path_exists = False
+        expected_path = os.path.join(self.basedir, squid.AUTH_FILE)
+        result = squid._get_auth_temp_path()
+        self.assertEqual(expected_path, result)
+        self.assertTrue(('fake_basedir_fn', '') in self.called)
+        self.assertTrue(('fake_makedirs', self.basedir) in self.called)
+        self.assertTrue(self.basedir in self.created_paths)
+        self.assertTrue(('fake_exists', self.basedir) in self.called)
+
+
+class EnvironTestCase(BaseTestCase):
+    """Test the different environ functions."""
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        """Set the tests."""
+        yield super(EnvironTestCase, self).setUp()
+        self.called = []
+        self.settings = dict(noauth_port=3434, auth_port=232323,
+                             username='u1', password='test')
+
+        def fake_dumps(data):
+            """Fake dumps."""
+            self.called.append(('dumps', data))
+            return json.dumps(data)
+
+        def fake_loads(data):
+            """Fake loads."""
+            self.called.append(('loads', data))
+            return json.loads(data)
+
+        self.patch(squid, 'dumps', fake_dumps)
+        self.patch(squid, 'loads', fake_loads)
+        self.env = {}
+        self.old_env = os.environ
+        squid.environ = self.env
+        self.addCleanup(self.set_back_environ)
+
+    def set_back_environ(self):
+        """Set back the env."""
+        squid.environ = self.old_env
+
+    def test_store_settings(self):
+        """Test the storage of the settings."""
+        squid.store_proxy_settings(self.settings)
+        self.assertTrue(('dumps', self.settings) in self.called)
+        self.assertEqual(self.env[squid.PROXY_ENV_VAR],
+                json.dumps(self.settings))
+
+    def test_retrieve_proxy_settings(self):
+        """Test reading the settings."""
+        self.env[squid.PROXY_ENV_VAR] = json.dumps(self.settings)
+        self.assertTrue(('loads', self.env[squid.PROXY_ENV_VAR]))
+        self.assertEqual(squid.retrieve_proxy_settings(), self.settings)
+
+    def test_delete_proxy_settings_present(self):
+        """Delete the proxy settings."""
+        self.env[squid.PROXY_ENV_VAR] = json.dumps(self.settings)
+        squid.delete_proxy_settings()
+        self.assertFalse(squid.PROXY_ENV_VAR in self.env)
+
+
+class SquidRunnerInitTestCase(BaseTestCase):
+    """Test the creation of the runner."""
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        """Set the different tests."""
+        yield super(SquidRunnerInitTestCase, self).setUp()
+        self.executables = {}
+        self.called = []
+
+        def fake_find_executable(executable):
+            """Fake the find executable."""
+            self.called.append(('fake_find_executable', executable))
+            return self.executables.get(executable, None)
+
+        self.patch(squid, 'find_executable', fake_find_executable)
+
+    def _assert_missing_binary(self, binary):
+        """Perform the assertion when a bin is missing."""
+        self.assertRaises(squid.SquidLaunchError, squid.SquidRunner)
+        self.assertTrue(('fake_find_executable', binary) in self.called,
+                self.called)
+
+    def test_squid_missing(self):
+        """Test when squid is missing."""
+        self.executables['htpasswd'] = 'htpasswd'
+        self._assert_missing_binary('squid')
+
+    def test_htpasswd_missing(self):
+        """Test when htpasswd is missing."""
+        self.executables['squid'] = 'squid'
+        self.executables['squid3'] = 'squid'
+        self._assert_missing_binary('htpasswd')
+
+
+class FakeSubprocess(object):
+    """Fake the subprocess module."""
+
+    # pylint: disable=C0103
+    def __init__(self):
+        """Create a new instance."""
+        self.called = []
+        self.PIPE = 'PIPE'
+
+    def Popen(self, args, stdout=None, stderr=None):
+        """Fake Popen."""
+        self.called.append(('Popen', args, stdout, stderr))
+        return self
+    # pylint: enable=C0103
+
+    def wait(self):
+        """Fake wait from a Popen object."""
+        self.called.append(('wait',))
+
+
+class FakeTemplate(object):
+    """Fake the string.Template."""
+
+    def __init__(self):
+        """Create a new instance."""
+        self.data = None
+        self.called = []
+
+    def __call__(self, data):
+        """Fake constructor."""
+        self.data = data
+        return self
+
+    def safe_substitute(self, *args, **kwargs):
+        """Fake the safe_substitute."""
+        self.called.append(('safe_substitute', args, kwargs))
+        return self.data
+
+
+class SquidRunnerTestCase(BaseTestCase):
+    """Test the default test case."""
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        """Set the different tests."""
+        yield super(SquidRunnerTestCase, self).setUp()
+        self.subprocess = FakeSubprocess()
+        self.patch(squid, 'subprocess', self.subprocess)
+
+        self.called = []
+        self.executables = dict(squid='squid', htpasswd='htpasswd')
+
+        def fake_find_executable(executable):
+            """Fake the find executable."""
+            self.called.append(('fake_find_executable', executable))
+            return self.executables.get(executable, None)
+
+        self.patch(squid, 'find_executable', fake_find_executable)
+
+        self.auth_temp = 'path/to/auth'
+
+        def fake_get_auth_temp_path(tempdir):
+            """Return the path for the auth file."""
+            self.called.append(('fake_get_auth_temp_path', tempdir))
+            return self.auth_temp
+
+        self.patch(squid, '_get_auth_temp_path', fake_get_auth_temp_path)
+
+        self.port = 2324
+
+        def fake_get_port():
+            """Fake the methos that returns the ports."""
+            self.called.append(('fake_get_port',))
+            return self.port
+
+        self.patch(squid, 'get_arbitrary_port', fake_get_port)
+        self.template = FakeTemplate()
+        self.patch(squid.string, 'Template', self.template)
+        self.runner = squid.SquidRunner()
+
+    def test_generate_swap(self):
+        """Test the generation of the squid swap."""
+        config_file = 'path/to/config'
+        expected_args = ['squid', '-z', '-f', config_file]
+        self.runner._generate_swap(config_file)
+        self.assertEqual(expected_args, self.subprocess.called[0][1])
+        self.assertTrue('wait' in self.subprocess.called[1])
+
+    def test_generate_auth_file(self):
+        """Test the generation of the auth file."""
+        username = self.runner.settings['username'] = 'mandel'
+        password = self.runner.settings['password'] = 'test'
+        expected_args = ['htpasswd', '-bc', self.auth_temp,
+                         username, password]
+        self.patch(squid, 'exists', lambda f: False)
+        self.runner._generate_auth_file()
+        self.assertEqual(expected_args, self.subprocess.called[0][1])
+        self.assertTrue('wait' in self.subprocess.called[1])
+
+    def test_generate_config_file(self):
+        """Test the generation of the config file."""
+        self.runner.auth_file = self.auth_temp
+        self.runner._generate_config_file(self.tmpdir)
+        # remove the generated file
+        self.addCleanup(os.unlink, self.runner.config_file)
+        expected_parameters = ('safe_substitute', (),
+                dict(auth_file=self.runner.auth_file,
+                     noauth_port_number=self.port,
+                     auth_port_number=self.port,
+                     spool_temp=squid._get_spool_temp_path(self.tmpdir),
+                     squid_temp=squid._get_squid_temp_path(self.tmpdir)))
+        self.assertTrue(expected_parameters, self.template.called[0])
+        self.assertEqual(2, self.called.count(('fake_get_port',)))

=== modified file 'ubuntuone/devtools/testcases/dbus.py'
--- ubuntuone/devtools/testcases/dbus.py	2011-11-23 11:14:00 +0000
+++ ubuntuone/devtools/testcases/dbus.py	2012-01-17 21:43:26 +0000
@@ -14,7 +14,7 @@
 # You should have received a copy of the GNU General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Base tests cases and test utilities."""
+"""Base dbus tests cases and test utilities."""
 
 from __future__ import absolute_import, with_statement
 

=== added file 'ubuntuone/devtools/testcases/squid.py'
--- ubuntuone/devtools/testcases/squid.py	1970-01-01 00:00:00 +0000
+++ ubuntuone/devtools/testcases/squid.py	2012-01-17 21:43:26 +0000
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Base squid tests cases and test utilities."""
+
+from ubuntuone.devtools.testcase import BaseTestCase, skipIf
+from ubuntuone.devtools.services.squid import (
+    SquidRunner,
+    SquidLaunchError,
+    get_squid_executable,
+    get_htpasswd_executable,
+    retrieve_proxy_settings)
+
+# pylint: disable=C0103
+squid, _ = get_squid_executable()
+htpasswd = get_htpasswd_executable()
+# pylint: enable=C0103
+
+
+@skipIf(squid is None or htpasswd is None,
+        'The test requires squid and htpasswd.')
+class SquidTestCase(BaseTestCase):
+    """Test that uses a proxy."""
+
+    def required_services(self):
+        """Return the list of required services for DBusTestCase."""
+        services = super(SquidTestCase, self).required_services()
+        services.extend([SquidRunner])
+        return services
+
+    def get_nonauth_proxy_settings(self):
+        """Return the settings of the noneauth proxy."""
+        settings = retrieve_proxy_settings()
+        if settings is None:
+            raise SquidLaunchError('Proxy is not running.')
+        return dict(host='localhost', port=settings['noauth_port'])
+
+    def get_auth_proxy_settings(self):
+        """Return the settings of the auth proxy."""
+        settings = retrieve_proxy_settings()
+        if settings is None:
+            raise SquidLaunchError('Proxy is not running.')
+        return dict(host='localhost', port=settings['auth_port'],
+                    username=settings['username'],
+                    password=settings['password'])

=== added directory 'ubuntuone/devtools/testcases/tests'
=== added file 'ubuntuone/devtools/testcases/tests/__init__.py'
--- ubuntuone/devtools/testcases/tests/__init__.py	1970-01-01 00:00:00 +0000
+++ ubuntuone/devtools/testcases/tests/__init__.py	2012-01-17 21:43:26 +0000
@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Tests for the test cases."""

=== added file 'ubuntuone/devtools/testcases/tests/test_squid_testcase.py'
--- ubuntuone/devtools/testcases/tests/test_squid_testcase.py	1970-01-01 00:00:00 +0000
+++ ubuntuone/devtools/testcases/tests/test_squid_testcase.py	2012-01-17 21:43:26 +0000
@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Test the squid test case."""
+import base64
+
+from twisted.application import internet, service
+from twisted.internet import defer, reactor
+from twisted.web import client, error, http, resource, server
+
+from ubuntuone.devtools.testcases.squid import SquidTestCase
+
+
+SAMPLE_RESOURCE = "<p>Hello World!</p>"
+SIMPLERESOURCE = "simpleresource"
+THROWERROR = "throwerror"
+UNAUTHORIZED = "unauthorized"
+
+# ignore common twisted lint errors
+# pylint: disable=C0103, W0212
+
+
+class ProxyClientFactory(client.HTTPClientFactory):
+    """Factory that supports proxy."""
+
+    def __init__(self, proxy_url, proxy_port, url, headers=None):
+        # we set the proxy details before the init because the parent __init__
+        # calls setURL
+        self.proxy_url = proxy_url
+        self.proxy_port = proxy_port
+        self.disconnected_d = defer.Deferred()
+        client.HTTPClientFactory.__init__(self, url, headers=headers)
+
+    def setURL(self, url):
+        self.host = self.proxy_url
+        self.port = self.proxy_port
+        self.url = url
+        self.path = url
+
+    def clientConnectionLost(self, connector, reason, reconnecting=0):
+        """Connection lost."""
+        self.disconnected_d.callback(self)
+
+
+class ProxyWebClient(object):
+    """Provide useful web methods with proxy."""
+
+    def __init__(self, proxy_url=None, proxy_port=None, username=None,
+            password=None):
+        """Create a new instance with the proxy settings."""
+        self.proxy_url = proxy_url
+        self.proxy_port = proxy_port
+        self.username = username
+        self.password = password
+        self.factory = None
+        self.connectors = []
+
+    def _connect(self, url, contextFactory):
+        """Perform the connection."""
+        scheme, _, _, _ = client._parse(url)
+        # pylint: disable=E1101
+        if scheme == 'https':
+            from twisted.internet import ssl
+            if contextFactory is None:
+                contextFactory = ssl.ClientContextFactory()
+            self.connectors.append(reactor.connectSSL(self.proxy_url,
+                                                      self.proxy_port,
+                                                      self.factory,
+                                                      contextFactory))
+        else:
+            self.connectors.append(reactor.connectTCP(self.proxy_url,
+                                                      self.proxy_port,
+                                                      self.factory))
+            # pylint: enable=E1101
+
+    def _process_auth_error(self, failure, url, contextFactory):
+        """Process an auth failure."""
+        failure.trap(error.Error)
+        if failure.value.status == str(http.PROXY_AUTH_REQUIRED):
+            # we try to get the page using the basic auth
+            auth = base64.b64encode('%s:%s' % (self.username, self.password))
+            auth_header = 'Basic ' + auth.strip()
+            self.factory = ProxyClientFactory(self.proxy_url, self.proxy_port,
+                            url, headers={'Proxy-Authorization': auth_header})
+            self._connect(url, contextFactory)
+            return self.factory.deferred
+        else:
+            return failure
+
+    def get_page(self, url, contextFactory=None, *args, **kwargs):
+        """Download a webpage as a string.
+
+        This method relies on the twisted.web.client.getPage but adds and extra
+        step. If there is an auth error the method will perform a second try
+        so that the username and password are used.
+        """
+        self.factory = ProxyClientFactory(self.proxy_url, self.proxy_port, url,
+                                          headers={'Connection': 'close'})
+        self._connect(url, contextFactory)
+        self.factory.deferred.addErrback(self._process_auth_error, url,
+                                    contextFactory)
+        return self.factory.deferred
+
+    @defer.inlineCallbacks
+    def shutdown(self):
+        """Clean all connectors."""
+        for connector in self.connectors:
+            yield connector.disconnect()
+        defer.returnValue(True)
+
+
+class SimpleResource(resource.Resource):
+    """A simple web resource."""
+
+    def render_GET(self, request):
+        """Make a bit of html out of these resource's
+        content."""
+        return SAMPLE_RESOURCE
+
+
+class SaveHTTPChannel(http.HTTPChannel):
+    """A save protocol to be used in tests."""
+
+    protocolInstance = None
+
+    def connectionMade(self):
+        """Keep track of the given protocol."""
+        SaveHTTPChannel.protocolInstance = self
+        http.HTTPChannel.connectionMade(self)
+
+
+class SaveSite(server.Site):
+    """A site that let us know when it closed."""
+
+    protocol = SaveHTTPChannel
+
+    def __init__(self, *args, **kwargs):
+        """Create a new instance."""
+        server.Site.__init__(self, *args, **kwargs)
+        # we disable the timeout in the tests, we will deal with it manually.
+        self.timeOut = None
+
+
+class MockWebServer(object):
+    """A mock webserver for testing"""
+
+    def __init__(self):
+        """Start up this instance."""
+        root = resource.Resource()
+        root.putChild(SIMPLERESOURCE, SimpleResource())
+
+        root.putChild(THROWERROR, resource.NoResource())
+
+        unauthorized_resource = resource.ErrorPage(resource.http.UNAUTHORIZED,
+                                                "Unauthorized", "Unauthorized")
+        root.putChild(UNAUTHORIZED, unauthorized_resource)
+
+        self.site = SaveSite(root)
+        application = service.Application('web')
+        self.service_collection = service.IServiceCollection(application)
+        #pylint: disable=E1101
+        self.tcpserver = internet.TCPServer(0, self.site)
+        self.tcpserver.setServiceParent(self.service_collection)
+        self.service_collection.startService()
+
+    def get_url(self):
+        """Build the url for this mock server."""
+        #pylint: disable=W0212
+        port_num = self.tcpserver._port.getHost().port
+        return "http://localhost:%d/" % port_num
+
+    @defer.inlineCallbacks
+    def stop(self):
+        """Shut it down."""
+        #pylint: disable=E1101
+        # make the connection time out so that is works with squid3 when
+        # the connection is kept alive.
+        if self.site.protocol.protocolInstance:
+            self.site.protocol.protocolInstance.timeoutConnection()
+        yield self.service_collection.stopService()
+
+
+class ProxyTestCase(SquidTestCase):
+    """A squid test with no auth proxy."""
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        """Set the tests."""
+        yield super(ProxyTestCase, self).setUp()
+        self.ws = MockWebServer()
+        self.proxy_client = None
+        self.addCleanup(self.teardown_client_server)
+        self.url = self.ws.get_url() + SIMPLERESOURCE
+
+    def teardown_client_server(self):
+        """Clean resources."""
+        if self.proxy_client is not None:
+            self.proxy_client.shutdown()
+            return defer.gatherResults([self.ws.stop(),
+                               self.proxy_client.shutdown(),
+                               self.proxy_client.factory.disconnected_d])
+        else:
+            return self.ws.stop()
+
+    def access_noauth_url(self, address, port):
+        """Access a url throught the proxy."""
+        self.proxy_client = ProxyWebClient(proxy_url=address, proxy_port=port)
+        return self.proxy_client.get_page(self.url)
+
+    def access_auth_url(self, address, port, username, password):
+        """Access a url throught the proxy."""
+        self.proxy_client = ProxyWebClient(proxy_url=address, proxy_port=port,
+                                         username=username, password=password)
+        return self.proxy_client.get_page(self.url)
+
+    @defer.inlineCallbacks
+    def test_noauth_url_access(self):
+        """Test accessing to the url."""
+        settings = self.get_nonauth_proxy_settings()
+        # if there is an exception we fail.
+        data = yield self.access_noauth_url(settings['host'],
+                                            settings['port'])
+        self.assertEqual(SAMPLE_RESOURCE, data)
+
+    @defer.inlineCallbacks
+    def test_auth_url_access(self):
+        """Test accessing to the url."""
+        settings = self.get_auth_proxy_settings()
+        # if there is an exception we fail.
+        data = yield self.access_auth_url(settings['host'],
+                                          settings['port'],
+                                          settings['username'],
+                                          settings['password'])
+        self.assertEqual(SAMPLE_RESOURCE, data)
+
+    def test_auth_url_401(self):
+        """Test failing accessing the url."""
+        settings = self.get_auth_proxy_settings()
+        # swap password for username to fail
+        d = self.failUnlessFailure(self.access_auth_url(settings['host'],
+                                        settings['port'], settings['password'],
+                                        settings['username']), error.Error)
+        return d
+
+    def test_auth_url_407(self):
+        """Test failing accessing the url."""
+        settings = self.get_auth_proxy_settings()
+        d = self.failUnlessFailure(self.access_noauth_url(settings['host'],
+                                   settings['port']), error.Error)
+        return d