Merge lp:~stub/launchpad/mock-swift into lp:launchpad
- mock-swift
- Merge into devel
Status: | Merged |
---|---|
Merged at revision: | 16762 |
Proposed branch: | lp:~stub/launchpad/mock-swift |
Merge into: | lp:launchpad |
Diff against target: |
335 lines (+230/-0) 6 files modified
lib/lp/testing/layers.py (+18/-0) lib/lp/testing/swift/fixture.py (+90/-0) lib/lp/testing/swift/hollow.tac (+33/-0) lib/lp/testing/swift/tests/test_fixture.py (+75/-0) setup.py (+3/-0) versions.cfg (+11/-0) |
To merge this branch: | bzr merge lp:~stub/launchpad/mock-swift |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Canonical Launchpad Engineering | Pending | ||
Review via email: mp+178047@code.launchpad.net |
Commit message
Description of the change
Swift test infrastructure
Stuart Bishop (stub) wrote : | # |
Stuart Bishop (stub) wrote : | # |
I've added the s4 tarball to the dependencies branch.
William Grant (wgrant) wrote : | # |
154 +import os
155 +import sys
156 +import logging
157 +
158 +from OpenSSL import crypto
os, sys and crypto are unused.
133 + def startup(self):
134 + self.setUp()
135 +
136 + def shutdown(self):
137 + self.cleanUp()
138 + while self._hasDaemon
139 + time.sleep(0.1)
Are these only here for the shutdown test? I'm not sure it's necessarily valuable to have them factored out, but it might also be worth pushing them up to TacTestSetup.
264 === modified file 'setup.py'
There should only be two new deps here; why are the indirect ones included directly? I imagine it might be for the setuptools-git pre_requires mess, but it's probably better to patch that out, as it's dreadfully fragile when there's no network access, and a security hole when there is network access.
Stuart Bishop (stub) wrote : | # |
I have fixed the imports.
Per IRC, the wait-until-
For the setup.py dependencies: s4 is needed by the test suite. mock is unused and I have removed it from setup.py and versions.cfg. python-swiftclient is needed by the new Librarian code. python-
Instead of using tarballs for python-swiftclient and python-keystone client I built .egg files locally and put them in the dependency branch. Buildout uses the .egg directly, skipping the step building the .egg from the tarball and avoiding network access.
Stuart Bishop (stub) wrote : | # |
Bug #1222711 for the txfixtures issue.
William Grant (wgrant) : | # |
Preview Diff
1 | === modified file 'lib/lp/testing/layers.py' |
2 | --- lib/lp/testing/layers.py 2013-06-20 05:50:00 +0000 |
3 | +++ lib/lp/testing/layers.py 2013-09-09 10:28:33 +0000 |
4 | @@ -38,6 +38,7 @@ |
5 | 'LibrarianLayer', |
6 | 'PageTestLayer', |
7 | 'RabbitMQLayer', |
8 | + 'SwiftLayer', |
9 | 'TwistedAppServerLayer', |
10 | 'TwistedLaunchpadZopelessLayer', |
11 | 'TwistedLayer', |
12 | @@ -147,6 +148,7 @@ |
13 | reset_logging, |
14 | ) |
15 | from lp.testing.pgsql import PgTestSetup |
16 | +from lp.testing.swift.fixture import SwiftFixture |
17 | from lp.testing.smtpd import SMTPController |
18 | |
19 | |
20 | @@ -822,6 +824,22 @@ |
21 | return cls._db_fixture.dropDb() |
22 | |
23 | |
24 | +class SwiftLayer(BaseLayer): |
25 | + @classmethod |
26 | + @profiled |
27 | + def setUp(cls): |
28 | + cls.swift_fixture = SwiftFixture() |
29 | + cls.swift_fixture.setUp() |
30 | + |
31 | + @classmethod |
32 | + @profiled |
33 | + def tearDown(cls): |
34 | + swift = cls.swift_fixture |
35 | + if swift is not None: |
36 | + cls.swift_fixture = None |
37 | + swift.cleanUp() |
38 | + |
39 | + |
40 | class LibrarianLayer(DatabaseLayer): |
41 | """Provides tests access to a Librarian instance. |
42 | |
43 | |
44 | === added directory 'lib/lp/testing/swift' |
45 | === added file 'lib/lp/testing/swift/__init__.py' |
46 | === added file 'lib/lp/testing/swift/fixture.py' |
47 | --- lib/lp/testing/swift/fixture.py 1970-01-01 00:00:00 +0000 |
48 | +++ lib/lp/testing/swift/fixture.py 2013-09-09 10:28:33 +0000 |
49 | @@ -0,0 +1,90 @@ |
50 | +# Copyright 2013 Canonical Ltd. This software is licensed under the |
51 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
52 | + |
53 | +"""Mock Swift test fixture.""" |
54 | + |
55 | +__metaclass__ = type |
56 | +__all__ = ['SwiftFixture'] |
57 | + |
58 | +import os.path |
59 | +import shutil |
60 | +import socket |
61 | +import tempfile |
62 | +import time |
63 | + |
64 | +from fixtures import FunctionFixture |
65 | +from s4 import hollow |
66 | +from swiftclient import client as swiftclient |
67 | +import testtools.content |
68 | +import testtools.content_type |
69 | +from txfixtures.tachandler import TacTestFixture |
70 | + |
71 | +from lp.services.config import config |
72 | + |
73 | + |
74 | +class SwiftFixture(TacTestFixture): |
75 | + |
76 | + tacfile = os.path.join(os.path.dirname(__file__), 'hollow.tac') |
77 | + pidfile = None |
78 | + logfile = None |
79 | + root = None |
80 | + daemon_port = None |
81 | + |
82 | + def setUp(self, spew=False, umask=None): |
83 | + # Pick a random, free port. |
84 | + if self.daemon_port is None: |
85 | + sock = socket.socket() |
86 | + sock.bind(('', 0)) |
87 | + self.daemon_port = sock.getsockname()[1] |
88 | + sock.close() |
89 | + self.logfile = os.path.join( |
90 | + config.root, 'logs', 'hollow-{}.log'.format(self.daemon_port)) |
91 | + self.pidfile = os.path.join( |
92 | + config.root, 'logs', 'hollow-{}.pid'.format(self.daemon_port)) |
93 | + assert self.daemon_port is not None |
94 | + |
95 | + super(SwiftFixture, self).setUp( |
96 | + spew, umask, |
97 | + os.path.join(config.root, 'bin', 'py'), |
98 | + os.path.join(config.root, 'bin', 'twistd')) |
99 | + |
100 | + def cleanUp(self): |
101 | + if self.logfile is not None and os.path.exists(self.logfile): |
102 | + self.addDetail( |
103 | + 'log-file', testtools.content.content_from_stream( |
104 | + open(self.logfile, 'r'), testtools.content_type.UTF8_TEXT)) |
105 | + os.unlink(self.logfile) |
106 | + super(SwiftFixture, self).cleanUp() |
107 | + |
108 | + def setUpRoot(self): |
109 | + # Create a root directory. |
110 | + if self.root is None or not os.path.isdir(self.root): |
111 | + root_fixture = FunctionFixture(tempfile.mkdtemp, shutil.rmtree) |
112 | + self.useFixture(root_fixture) |
113 | + self.root = root_fixture.fn_result |
114 | + os.chmod(self.root, 0o700) |
115 | + assert os.path.isdir(self.root) |
116 | + |
117 | + # Pass on options to the daemon. |
118 | + os.environ['HOLLOW_ROOT'] = self.root |
119 | + os.environ['HOLLOW_PORT'] = str(self.daemon_port) |
120 | + |
121 | + def connect( |
122 | + self, tenant_name=hollow.DEFAULT_TENANT_NAME, |
123 | + username=hollow.DEFAULT_USERNAME, password=hollow.DEFAULT_PASSWORD): |
124 | + """Return a valid connection to our mock Swift""" |
125 | + port = self.daemon_port |
126 | + client = swiftclient.Connection( |
127 | + authurl="http://localhost:%d/keystone/v2.0/" % port, |
128 | + auth_version="2.0", tenant_name=tenant_name, |
129 | + user=username, key=password, |
130 | + retries=0, insecure=True) |
131 | + return client |
132 | + |
133 | + def startup(self): |
134 | + self.setUp() |
135 | + |
136 | + def shutdown(self): |
137 | + self.cleanUp() |
138 | + while self._hasDaemonStarted(): |
139 | + time.sleep(0.1) |
140 | |
141 | === added file 'lib/lp/testing/swift/hollow.tac' |
142 | --- lib/lp/testing/swift/hollow.tac 1970-01-01 00:00:00 +0000 |
143 | +++ lib/lp/testing/swift/hollow.tac 2013-09-09 10:28:33 +0000 |
144 | @@ -0,0 +1,33 @@ |
145 | +# -*- python -*- |
146 | +# Copyright 2013 Canonical Ltd. This software is licensed under the |
147 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
148 | + |
149 | +'''Launch a mock Swift service.''' |
150 | + |
151 | +__metaclass__ = type |
152 | +__all__ = [] |
153 | + |
154 | +import os.path |
155 | +import logging |
156 | + |
157 | +import twisted.web.server |
158 | +from twisted.application import internet, service |
159 | + |
160 | +logging.basicConfig() |
161 | + |
162 | +from s4 import hollow |
163 | + |
164 | +storedir = os.environ['HOLLOW_ROOT'] |
165 | +assert os.path.exists(storedir) |
166 | + |
167 | +application = service.Application('hollow') |
168 | +root = hollow.Root(storage_dir=storedir, hostname='localhost') |
169 | + |
170 | +# make sure "the bucket" is created |
171 | +root.swift.addBucket("the bucket") |
172 | +site = twisted.web.server.Site(root) |
173 | + |
174 | +port = int(os.environ['HOLLOW_PORT']) |
175 | + |
176 | +sc = service.IServiceCollection(application) |
177 | +internet.TCPServer(port, site).setServiceParent(sc) |
178 | |
179 | === added directory 'lib/lp/testing/swift/tests' |
180 | === added file 'lib/lp/testing/swift/tests/__init__.py' |
181 | === added file 'lib/lp/testing/swift/tests/test_fixture.py' |
182 | --- lib/lp/testing/swift/tests/test_fixture.py 1970-01-01 00:00:00 +0000 |
183 | +++ lib/lp/testing/swift/tests/test_fixture.py 2013-09-09 10:28:33 +0000 |
184 | @@ -0,0 +1,75 @@ |
185 | +# Copyright 2013 Canonical Ltd. This software is licensed under the |
186 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
187 | + |
188 | +"""Testing the mock Swift test fixture.""" |
189 | + |
190 | +__metaclass__ = type |
191 | +__all__ = [] |
192 | + |
193 | +import httplib |
194 | + |
195 | +from swiftclient import client as swiftclient |
196 | + |
197 | +from lp.testing import TestCase |
198 | +from lp.testing.layers import BaseLayer |
199 | +from lp.testing.swift.fixture import SwiftFixture |
200 | + |
201 | + |
202 | +class TestSwiftFixture(TestCase): |
203 | + layer = BaseLayer |
204 | + |
205 | + def setUp(self): |
206 | + super(TestSwiftFixture, self).setUp() |
207 | + self.swift_fixture = SwiftFixture() |
208 | + self.useFixture(self.swift_fixture) |
209 | + |
210 | + def test_basic(self): |
211 | + client = self.swift_fixture.connect() |
212 | + size = 30 |
213 | + headers, body = client.get_object("size", str(size)) |
214 | + self.assertEquals(body, "0" * size) |
215 | + self.assertEqual(str(size), headers["content-length"]) |
216 | + self.assertEqual("text/plain", headers["content-type"]) |
217 | + |
218 | + def test_shutdown_and_startup(self): |
219 | + # This test demonstrates how the Swift client deals with a |
220 | + # flapping Swift server. In particular, that once a connection |
221 | + # has started failing it will continue failing so we need to |
222 | + # ensure that once we encounter a fail we open a fresh |
223 | + # connection. This is probably a property of our mock Swift |
224 | + # server rather than reality but the mock is a required target. |
225 | + size = 30 |
226 | + |
227 | + # With no Swift server, a fresh connection fails with |
228 | + # a swiftclient.ClientException when it fails to |
229 | + # authenticate. |
230 | + self.swift_fixture.shutdown() |
231 | + client = self.swift_fixture.connect() |
232 | + self.assertRaises( |
233 | + swiftclient.ClientException, |
234 | + client.get_object, "size", str(size)) |
235 | + |
236 | + # Things work fine when the Swift server is up. |
237 | + self.swift_fixture.startup() |
238 | + headers, body = client.get_object("size", str(size)) |
239 | + self.assertEquals(body, "0" * size) |
240 | + |
241 | + # But if the Swift server goes away again, we end up with |
242 | + # different failures since the connection has already |
243 | + # authenticated. |
244 | + self.swift_fixture.shutdown() |
245 | + self.assertRaises( |
246 | + httplib.HTTPException, |
247 | + client.get_object, "size", str(size)) |
248 | + |
249 | + # And even if we bring it back up, existing connections |
250 | + # continue to fail |
251 | + self.swift_fixture.startup() |
252 | + self.assertRaises( |
253 | + httplib.HTTPException, |
254 | + client.get_object, "size", str(size)) |
255 | + |
256 | + # But fresh connections are fine. |
257 | + client = self.swift_fixture.connect() |
258 | + headers, body = client.get_object("size", str(size)) |
259 | + self.assertEquals(body, "0" * size) |
260 | |
261 | === modified file 'setup.py' |
262 | --- setup.py 2013-04-15 07:03:38 +0000 |
263 | +++ setup.py 2013-09-09 10:28:33 +0000 |
264 | @@ -41,7 +41,9 @@ |
265 | 'html5browser', |
266 | 'pygpgme', |
267 | 'python-debian', |
268 | + 'python-keystoneclient', |
269 | 'python-subunit', |
270 | + 'python-swiftclient', |
271 | 'launchpadlib', |
272 | 'lazr.batchnavigator', |
273 | 'lazr.config', |
274 | @@ -74,6 +76,7 @@ |
275 | 'python-openid', |
276 | 'pytz', |
277 | 'rabbitfixture', |
278 | + 's4', |
279 | 'setproctitle', |
280 | 'setuptools', |
281 | 'Sphinx', |
282 | |
283 | === modified file 'versions.cfg' |
284 | --- versions.cfg 2013-08-16 05:15:35 +0000 |
285 | +++ versions.cfg 2013-09-09 10:28:33 +0000 |
286 | @@ -20,6 +20,7 @@ |
287 | celery = 2.5.1 |
288 | Chameleon = 2.11 |
289 | cssutils = 0.9.10 |
290 | +d2to1 = 0.2.10 |
291 | Django = 1.4 |
292 | # Required by pydkim |
293 | dnspython = 1.10.0 |
294 | @@ -66,8 +67,12 @@ |
295 | oops-twisted = 0.0.6 |
296 | oops-wsgi = 0.0.8 |
297 | ordereddict = 1.1 |
298 | +oslo.config = 1.1.1 |
299 | paramiko = 1.7.7.2 |
300 | +pbr = 0.5.20 |
301 | pgbouncer = 0.0.8 |
302 | +pip = 1.4 |
303 | +prettytable = 0.7.2 |
304 | psycopg2 = 2.4.6 |
305 | pyasn1 = 0.1.6 |
306 | pycrypto = 2.6 |
307 | @@ -80,6 +85,7 @@ |
308 | pystache = 0.5.3 |
309 | python-dateutil = 1.5 |
310 | python-debian = 0.1.21 |
311 | +python-keystoneclient = 0.3.1 |
312 | # lp:python-memcached |
313 | # r57 (includes a fix for bug 974632) |
314 | # bzr branch lp:python-memcached -r57 memcached-1.49 |
315 | @@ -94,15 +100,20 @@ |
316 | # reapplied a patch from wgrant to get codehosting going again. |
317 | python-openid = 2.2.5-fix1034376 |
318 | python-subunit = 0.0.8beta |
319 | +python-swiftclient = 1.5.0 |
320 | # lp-1 Build of lp:~benji/rabbitfixture/longer-timeout revision: r30 |
321 | # lp-2 Build of lp:~frankban/rabbitfixture/longer-timeout revision: r32 |
322 | # lp-3 Build of lp:~frankban/rabbitfixture/lp3 revision: r33 |
323 | # lp-4 Build of lp:~frankban/rabbitfixture/lp4 revision: r34 |
324 | # to build: python setup.py sdist |
325 | rabbitfixture = 0.3.3-lp-4 |
326 | +requests = 1.2.3 |
327 | +s4 = 0.1.1 |
328 | setproctitle = 1.1.7 |
329 | +setuptools-git = 1.0 |
330 | simplejson = 3.1.3 |
331 | SimpleTAL = 4.3 |
332 | +six = 1.3.0 |
333 | soupmatchers = 0.2 |
334 | sourcecodegen = 0.6.14 |
335 | # lp:~launchpad-committers/storm/with-without-datetime |
Blocked landing for now, as there is still one dependency to add to lp-sourcedeps (s4 has not yet been publicly released).