Merge lp:~stub/launchpad/mock-swift into lp:launchpad

[Stuart Bishop]

Swift test infrastructure

[Stuart Bishop (stub)]

Blocked landing for now, as there is still one dependency to add to lp-sourcedeps (s4 has not yet been publicly released).

[Stuart Bishop (stub)]

I've added the s4 tarball to the dependencies branch.

[William Grant (wgrant)]

154 +import os
155 +import sys
156 +import logging
157 +
158 +from OpenSSL import crypto

os, sys and crypto are unused.

133 + def startup(self):
134 + self.setUp()
135 +
136 + def shutdown(self):
137 + self.cleanUp()
138 + while self._hasDaemonStarted():
139 + time.sleep(0.1)

Are these only here for the shutdown test? I'm not sure it's necessarily valuable to have them factored out, but it might also be worth pushing them up to TacTestSetup.

264 === modified file 'setup.py'

There should only be two new deps here; why are the indirect ones included directly? I imagine it might be for the setuptools-git pre_requires mess, but it's probably better to patch that out, as it's dreadfully fragile when there's no network access, and a security hole when there is network access.

review: Needs Fixing (code)

[Stuart Bishop (stub)]

I have fixed the imports.

Per IRC, the wait-until-really-shutdown is needed for tests confirming things work correctly when Swift is dead. I'll submit a bug report on txfixtures - it can be argued this behavior belongs in the base fixture's cleanUp, although for general use that might need to be combined with a timeout fixture.

For the setup.py dependencies: s4 is needed by the test suite. mock is unused and I have removed it from setup.py and versions.cfg. python-swiftclient is needed by the new Librarian code. python-keystoneclient is needed because we are using v2.0 authentication, and python-swiftclient bombs out at runtime if you attempt to do this without the keystoneclient module installed.

Instead of using tarballs for python-swiftclient and python-keystone client I built .egg files locally and put them in the dependency branch. Buildout uses the .egg directly, skipping the step building the .egg from the tarball and avoiding network access.

[Stuart Bishop (stub)]

Bug #1222711 for the txfixtures issue.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'lib/lp/testing/layers.py'
--- lib/lp/testing/layers.py	2013-06-20 05:50:00 +0000
+++ lib/lp/testing/layers.py	2013-09-09 10:28:33 +0000
@@ -38,6 +38,7 @@
     'LibrarianLayer',
     'PageTestLayer',
     'RabbitMQLayer',
+    'SwiftLayer',
     'TwistedAppServerLayer',
     'TwistedLaunchpadZopelessLayer',
     'TwistedLayer',
@@ -147,6 +148,7 @@
     reset_logging,
     )
 from lp.testing.pgsql import PgTestSetup
+from lp.testing.swift.fixture import SwiftFixture
 from lp.testing.smtpd import SMTPController
 
 
@@ -822,6 +824,22 @@
         return cls._db_fixture.dropDb()
 
 
+class SwiftLayer(BaseLayer):
+    @classmethod
+    @profiled
+    def setUp(cls):
+        cls.swift_fixture = SwiftFixture()
+        cls.swift_fixture.setUp()
+
+    @classmethod
+    @profiled
+    def tearDown(cls):
+        swift = cls.swift_fixture
+        if swift is not None:
+            cls.swift_fixture = None
+            swift.cleanUp()
+
+
 class LibrarianLayer(DatabaseLayer):
     """Provides tests access to a Librarian instance.
 
 
=== added directory 'lib/lp/testing/swift'
=== added file 'lib/lp/testing/swift/__init__.py'
=== added file 'lib/lp/testing/swift/fixture.py'
--- lib/lp/testing/swift/fixture.py	1970-01-01 00:00:00 +0000
+++ lib/lp/testing/swift/fixture.py	2013-09-09 10:28:33 +0000
@@ -0,0 +1,90 @@
+# Copyright 2013 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Mock Swift test fixture."""
+
+__metaclass__ = type
+__all__ = ['SwiftFixture']
+
+import os.path
+import shutil
+import socket
+import tempfile
+import time
+
+from fixtures import FunctionFixture
+from s4 import hollow
+from swiftclient import client as swiftclient
+import testtools.content
+import testtools.content_type
+from txfixtures.tachandler import TacTestFixture
+
+from lp.services.config import config
+
+
+class SwiftFixture(TacTestFixture):
+
+    tacfile = os.path.join(os.path.dirname(__file__), 'hollow.tac')
+    pidfile = None
+    logfile = None
+    root = None
+    daemon_port = None
+
+    def setUp(self, spew=False, umask=None):
+        # Pick a random, free port.
+        if self.daemon_port is None:
+            sock = socket.socket()
+            sock.bind(('', 0))
+            self.daemon_port = sock.getsockname()[1]
+            sock.close()
+            self.logfile = os.path.join(
+                config.root, 'logs', 'hollow-{}.log'.format(self.daemon_port))
+            self.pidfile = os.path.join(
+                config.root, 'logs', 'hollow-{}.pid'.format(self.daemon_port))
+        assert self.daemon_port is not None
+
+        super(SwiftFixture, self).setUp(
+            spew, umask,
+            os.path.join(config.root, 'bin', 'py'),
+            os.path.join(config.root, 'bin', 'twistd'))
+
+    def cleanUp(self):
+        if self.logfile is not None and os.path.exists(self.logfile):
+            self.addDetail(
+                'log-file', testtools.content.content_from_stream(
+                    open(self.logfile, 'r'), testtools.content_type.UTF8_TEXT))
+            os.unlink(self.logfile)
+        super(SwiftFixture, self).cleanUp()
+
+    def setUpRoot(self):
+        # Create a root directory.
+        if self.root is None or not os.path.isdir(self.root):
+            root_fixture = FunctionFixture(tempfile.mkdtemp, shutil.rmtree)
+            self.useFixture(root_fixture)
+            self.root = root_fixture.fn_result
+            os.chmod(self.root, 0o700)
+        assert os.path.isdir(self.root)
+
+        # Pass on options to the daemon.
+        os.environ['HOLLOW_ROOT'] = self.root
+        os.environ['HOLLOW_PORT'] = str(self.daemon_port)
+
+    def connect(
+        self, tenant_name=hollow.DEFAULT_TENANT_NAME,
+        username=hollow.DEFAULT_USERNAME, password=hollow.DEFAULT_PASSWORD):
+        """Return a valid connection to our mock Swift"""
+        port = self.daemon_port
+        client = swiftclient.Connection(
+            authurl="http://localhost:%d/keystone/v2.0/" % port,
+            auth_version="2.0", tenant_name=tenant_name,
+            user=username, key=password,
+            retries=0, insecure=True)
+        return client
+
+    def startup(self):
+        self.setUp()
+
+    def shutdown(self):
+        self.cleanUp()
+        while self._hasDaemonStarted():
+            time.sleep(0.1)
 
=== added file 'lib/lp/testing/swift/hollow.tac'
--- lib/lp/testing/swift/hollow.tac	1970-01-01 00:00:00 +0000
+++ lib/lp/testing/swift/hollow.tac	2013-09-09 10:28:33 +0000
@@ -0,0 +1,33 @@
+# -*- python -*-
+# Copyright 2013 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+'''Launch a mock Swift service.'''
+
+__metaclass__ = type
+__all__ = []
+
+import os.path
+import logging
+
+import twisted.web.server
+from twisted.application import internet, service
+
+logging.basicConfig()
+
+from s4 import hollow
+
+storedir = os.environ['HOLLOW_ROOT']
+assert os.path.exists(storedir)
+
+application = service.Application('hollow')
+root = hollow.Root(storage_dir=storedir, hostname='localhost')
+
+# make sure "the bucket" is created
+root.swift.addBucket("the bucket")
+site = twisted.web.server.Site(root)
+
+port = int(os.environ['HOLLOW_PORT'])
+
+sc = service.IServiceCollection(application)
+internet.TCPServer(port, site).setServiceParent(sc)
 
=== added directory 'lib/lp/testing/swift/tests'
=== added file 'lib/lp/testing/swift/tests/__init__.py'
=== added file 'lib/lp/testing/swift/tests/test_fixture.py'
--- lib/lp/testing/swift/tests/test_fixture.py	1970-01-01 00:00:00 +0000
+++ lib/lp/testing/swift/tests/test_fixture.py	2013-09-09 10:28:33 +0000
@@ -0,0 +1,75 @@
+# Copyright 2013 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Testing the mock Swift test fixture."""
+
+__metaclass__ = type
+__all__ = []
+
+import httplib
+
+from swiftclient import client as swiftclient
+
+from lp.testing import TestCase
+from lp.testing.layers import BaseLayer
+from lp.testing.swift.fixture import SwiftFixture
+
+
+class TestSwiftFixture(TestCase):
+    layer = BaseLayer
+
+    def setUp(self):
+        super(TestSwiftFixture, self).setUp()
+        self.swift_fixture = SwiftFixture()
+        self.useFixture(self.swift_fixture)
+
+    def test_basic(self):
+        client = self.swift_fixture.connect()
+        size = 30
+        headers, body = client.get_object("size", str(size))
+        self.assertEquals(body, "0" * size)
+        self.assertEqual(str(size), headers["content-length"])
+        self.assertEqual("text/plain", headers["content-type"])
+
+    def test_shutdown_and_startup(self):
+        # This test demonstrates how the Swift client deals with a
+        # flapping Swift server. In particular, that once a connection
+        # has started failing it will continue failing so we need to
+        # ensure that once we encounter a fail we open a fresh
+        # connection. This is probably a property of our mock Swift
+        # server rather than reality but the mock is a required target.
+        size = 30
+
+        # With no Swift server, a fresh connection fails with
+        # a swiftclient.ClientException when it fails to
+        # authenticate.
+        self.swift_fixture.shutdown()
+        client = self.swift_fixture.connect()
+        self.assertRaises(
+            swiftclient.ClientException,
+            client.get_object, "size", str(size))
+
+        # Things work fine when the Swift server is up.
+        self.swift_fixture.startup()
+        headers, body = client.get_object("size", str(size))
+        self.assertEquals(body, "0" * size)
+
+        # But if the Swift server goes away again, we end up with
+        # different failures since the connection has already
+        # authenticated.
+        self.swift_fixture.shutdown()
+        self.assertRaises(
+            httplib.HTTPException,
+            client.get_object, "size", str(size))
+
+        # And even if we bring it back up, existing connections
+        # continue to fail
+        self.swift_fixture.startup()
+        self.assertRaises(
+            httplib.HTTPException,
+            client.get_object, "size", str(size))
+
+        # But fresh connections are fine.
+        client = self.swift_fixture.connect()
+        headers, body = client.get_object("size", str(size))
+        self.assertEquals(body, "0" * size)
 
=== modified file 'setup.py'
--- setup.py	2013-04-15 07:03:38 +0000
+++ setup.py	2013-09-09 10:28:33 +0000
@@ -41,7 +41,9 @@
         'html5browser',
         'pygpgme',
         'python-debian',
+        'python-keystoneclient',
         'python-subunit',
+        'python-swiftclient',
         'launchpadlib',
         'lazr.batchnavigator',
         'lazr.config',
@@ -74,6 +76,7 @@
         'python-openid',
         'pytz',
         'rabbitfixture',
+        's4',
         'setproctitle',
         'setuptools',
         'Sphinx',
 
=== modified file 'versions.cfg'
--- versions.cfg	2013-08-16 05:15:35 +0000
+++ versions.cfg	2013-09-09 10:28:33 +0000
@@ -20,6 +20,7 @@
 celery = 2.5.1
 Chameleon = 2.11
 cssutils = 0.9.10
+d2to1 = 0.2.10
 Django = 1.4
 # Required by pydkim
 dnspython = 1.10.0
@@ -66,8 +67,12 @@
 oops-twisted = 0.0.6
 oops-wsgi = 0.0.8
 ordereddict = 1.1
+oslo.config = 1.1.1
 paramiko = 1.7.7.2
+pbr = 0.5.20
 pgbouncer = 0.0.8
+pip = 1.4
+prettytable = 0.7.2
 psycopg2 = 2.4.6
 pyasn1 = 0.1.6
 pycrypto = 2.6
@@ -80,6 +85,7 @@
 pystache = 0.5.3
 python-dateutil = 1.5
 python-debian = 0.1.21
+python-keystoneclient = 0.3.1
 # lp:python-memcached
 # r57 (includes a fix for bug 974632)
 # bzr branch lp:python-memcached -r57 memcached-1.49
@@ -94,15 +100,20 @@
 # reapplied a patch from wgrant to get codehosting going again.
 python-openid = 2.2.5-fix1034376
 python-subunit = 0.0.8beta
+python-swiftclient = 1.5.0
 # lp-1 Build of lp:~benji/rabbitfixture/longer-timeout revision: r30
 # lp-2 Build of lp:~frankban/rabbitfixture/longer-timeout revision: r32
 # lp-3 Build of lp:~frankban/rabbitfixture/lp3 revision: r33
 # lp-4 Build of lp:~frankban/rabbitfixture/lp4 revision: r34
 # to build: python setup.py sdist
 rabbitfixture = 0.3.3-lp-4
+requests = 1.2.3
+s4 = 0.1.1
 setproctitle = 1.1.7
+setuptools-git = 1.0
 simplejson = 3.1.3
 SimpleTAL = 4.3
+six = 1.3.0
 soupmatchers = 0.2
 sourcecodegen = 0.6.14
 # lp:~launchpad-committers/storm/with-without-datetime