Merge lp:~wgrant/launchpad/more-zopeless-destruction into lp:launchpad
- more-zopeless-destruction
- Merge into devel
Status: | Merged |
---|---|
Approved by: | Stuart Bishop |
Approved revision: | no longer in the source branch. |
Merged at revision: | 13935 |
Proposed branch: | lp:~wgrant/launchpad/more-zopeless-destruction |
Merge into: | lp:launchpad |
Prerequisite: | lp:~wgrant/launchpad/destroy-lots-of-db-cruft |
Diff against target: |
682 lines (+42/-455) 9 files modified
lib/canonical/database/sqlbase.py (+11/-46) lib/canonical/lp/__init__.py (+2/-3) lib/canonical/lp/ftests/test_zopeless.py (+13/-15) lib/canonical/testing/layers.py (+2/-2) lib/lp/soyuz/scripts/tests/test_queue.py (+1/-4) lib/lp/soyuz/tests/soyuz.py (+8/-11) lib/lp/soyuz/tests/test_doc.py (+2/-6) lib/lp/soyuz/tests/test_packagediff.py (+3/-4) utilities/check-sampledata.py (+0/-364) |
To merge this branch: | bzr merge lp:~wgrant/launchpad/more-zopeless-destruction |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Stuart Bishop (community) | Approve | ||
Review via email: mp+74367@code.launchpad.net |
Commit message
Start stripping down ZopelessTransac
Description of the change
This branch continues to strip down the Zopeless APIs, preparing for its eventual destruction.
initZopeless no longer takes dbname/host arguments, bringing it down to just overriding user and isolation. Its internals are now simplified, as it doesn't have to mutate rw_main_master. And since dbuser is mandatory these days, the config overlay is now constructed in one hit.
Only one callsite, utilities/
All LaunchpadZopele
Preview Diff
1 | === modified file 'lib/canonical/database/sqlbase.py' |
2 | --- lib/canonical/database/sqlbase.py 2011-09-07 23:29:26 +0000 |
3 | +++ lib/canonical/database/sqlbase.py 2011-09-07 23:29:27 +0000 |
4 | @@ -291,39 +291,11 @@ |
5 | "directly instantiated.") |
6 | |
7 | @classmethod |
8 | - def _get_zopeless_connection_config(self, dbname, dbhost): |
9 | - # This method exists for testability. |
10 | - |
11 | - # This is only used by scripts, so we must connect to the read-write |
12 | - # DB here -- that's why we use rw_main_master directly. |
13 | - from canonical.database.postgresql import ConnectionString |
14 | - main_connection_string = ConnectionString(dbconfig.rw_main_master) |
15 | - |
16 | - # Override dbname and dbhost in the connection string if they |
17 | - # have been passed in. |
18 | - if dbname is None: |
19 | - dbname = main_connection_string.dbname |
20 | - else: |
21 | - main_connection_string.dbname = dbname |
22 | - |
23 | - if dbhost is None: |
24 | - dbhost = main_connection_string.host |
25 | - else: |
26 | - main_connection_string.host = dbhost |
27 | - |
28 | - return str(main_connection_string), dbname, dbhost |
29 | - |
30 | - @classmethod |
31 | - def initZopeless(cls, dbname=None, dbhost=None, dbuser=None, |
32 | - isolation=ISOLATION_LEVEL_DEFAULT): |
33 | - |
34 | - main_connection_string, dbname, dbhost = ( |
35 | - cls._get_zopeless_connection_config(dbname, dbhost)) |
36 | - |
37 | - assert dbuser is not None, ''' |
38 | - dbuser is now required. All scripts must connect as unique |
39 | - database users. |
40 | - ''' |
41 | + def initZopeless(cls, dbuser=None, isolation=ISOLATION_LEVEL_DEFAULT): |
42 | + if dbuser is None: |
43 | + raise AssertionError( |
44 | + "dbuser is now required. All scripts must connect as unique " |
45 | + "database users.") |
46 | |
47 | isolation_level = { |
48 | ISOLATION_LEVEL_AUTOCOMMIT: 'autocommit', |
49 | @@ -333,18 +305,13 @@ |
50 | # Construct a config fragment: |
51 | overlay = dedent("""\ |
52 | [database] |
53 | - rw_main_master: %(main_connection_string)s |
54 | isolation_level: %(isolation_level)s |
55 | - """ % { |
56 | - 'isolation_level': isolation_level, |
57 | - 'main_connection_string': main_connection_string, |
58 | - }) |
59 | |
60 | - if dbuser: |
61 | - overlay += dedent("""\ |
62 | - [launchpad] |
63 | - dbuser: %(dbuser)s |
64 | - """ % {'dbuser': dbuser}) |
65 | + [launchpad] |
66 | + dbuser: %(dbuser)s |
67 | + """ % dict( |
68 | + isolation_level=isolation_level, |
69 | + dbuser=dbuser)) |
70 | |
71 | if cls._installed is not None: |
72 | if cls._config_overlay != overlay: |
73 | @@ -357,8 +324,6 @@ |
74 | else: |
75 | config.push(cls._CONFIG_OVERLAY_NAME, overlay) |
76 | cls._config_overlay = overlay |
77 | - cls._dbname = dbname |
78 | - cls._dbhost = dbhost |
79 | cls._dbuser = dbuser |
80 | cls._isolation = isolation |
81 | cls._reset_stores() |
82 | @@ -419,7 +384,7 @@ |
83 | assert cls._installed is not None, ( |
84 | "ZopelessTransactionManager not installed") |
85 | cls.uninstall() |
86 | - cls.initZopeless(cls._dbname, cls._dbhost, cls._dbuser, isolation) |
87 | + cls.initZopeless(cls._dbuser, isolation) |
88 | |
89 | @staticmethod |
90 | def conn(): |
91 | |
92 | === modified file 'lib/canonical/lp/__init__.py' |
93 | --- lib/canonical/lp/__init__.py 2011-09-07 23:29:26 +0000 |
94 | +++ lib/canonical/lp/__init__.py 2011-09-07 23:29:27 +0000 |
95 | @@ -28,12 +28,11 @@ |
96 | return ZopelessTransactionManager._installed is not None |
97 | |
98 | |
99 | -def initZopeless(dbname=None, dbhost=None, dbuser=None, |
100 | - isolation=ISOLATION_LEVEL_DEFAULT): |
101 | +def initZopeless(dbuser=None, isolation=ISOLATION_LEVEL_DEFAULT): |
102 | """Initialize the Zopeless environment.""" |
103 | if dbuser is None: |
104 | dbuser = ( |
105 | ConnectionString(dbconfig.main_master).user or dbconfig.dbuser) |
106 | |
107 | return ZopelessTransactionManager.initZopeless( |
108 | - dbname=dbname, dbhost=dbhost, dbuser=dbuser, isolation=isolation) |
109 | + dbuser=dbuser, isolation=isolation) |
110 | |
111 | === modified file 'lib/canonical/lp/ftests/test_zopeless.py' |
112 | --- lib/canonical/lp/ftests/test_zopeless.py 2010-10-17 05:02:20 +0000 |
113 | +++ lib/canonical/lp/ftests/test_zopeless.py 2011-09-07 23:29:27 +0000 |
114 | @@ -13,12 +13,14 @@ |
115 | import psycopg2 |
116 | from sqlobject import StringCol, IntCol |
117 | |
118 | -from canonical.database.sqlbase import SQLBase, alreadyInstalledMsg, cursor |
119 | +from canonical.database.sqlbase import ( |
120 | + alreadyInstalledMsg, |
121 | + connect, |
122 | + cursor, |
123 | + SQLBase, |
124 | + ) |
125 | from canonical.lp import initZopeless |
126 | -from canonical.testing.layers import ( |
127 | - DatabaseLayer, |
128 | - LaunchpadScriptLayer, |
129 | - ) |
130 | +from canonical.testing.layers import LaunchpadScriptLayer |
131 | |
132 | |
133 | class MoreBeer(SQLBase): |
134 | @@ -50,11 +52,8 @@ |
135 | # Calling initZopeless with the same arguments twice should return |
136 | # the exact same object twice, but also emit a warning. |
137 | try: |
138 | - dbname = DatabaseLayer._db_fixture.dbname |
139 | - tm1 = initZopeless( |
140 | - dbname=dbname, dbhost='', dbuser='launchpad') |
141 | - tm2 = initZopeless( |
142 | - dbname=dbname, dbhost='', dbuser='launchpad') |
143 | + tm1 = initZopeless(dbuser='launchpad') |
144 | + tm2 = initZopeless(dbuser='launchpad') |
145 | self.failUnless(tm1 is tm2) |
146 | self.failUnless(self.warned) |
147 | finally: |
148 | @@ -73,8 +72,7 @@ |
149 | layer = LaunchpadScriptLayer |
150 | |
151 | def setUp(self): |
152 | - self.tm = initZopeless(dbname=DatabaseLayer._db_fixture.dbname, |
153 | - dbuser='launchpad') |
154 | + self.tm = initZopeless(dbuser='launchpad') |
155 | |
156 | c = cursor() |
157 | c.execute("CREATE TABLE morebeer (" |
158 | @@ -187,7 +185,7 @@ |
159 | self.tm.commit() |
160 | |
161 | # Make another change from a non-SQLObject connection, and commit that |
162 | - conn = psycopg2.connect('dbname=' + DatabaseLayer._db_fixture.dbname) |
163 | + conn = connect() |
164 | cur = conn.cursor() |
165 | cur.execute("BEGIN TRANSACTION;") |
166 | cur.execute("UPDATE MoreBeer SET rating=4 " |
167 | @@ -207,8 +205,7 @@ |
168 | >>> isZopeless() |
169 | False |
170 | |
171 | - >>> tm = initZopeless(dbname=DatabaseLayer._db_fixture.dbname, |
172 | - ... dbhost='', dbuser='launchpad') |
173 | + >>> tm = initZopeless(dbuser='launchpad') |
174 | >>> isZopeless() |
175 | True |
176 | |
177 | @@ -218,6 +215,7 @@ |
178 | |
179 | """ |
180 | |
181 | + |
182 | def test_suite(): |
183 | suite = unittest.TestSuite() |
184 | suite.addTest(unittest.makeSuite(TestZopeless)) |
185 | |
186 | === modified file 'lib/canonical/testing/layers.py' |
187 | --- lib/canonical/testing/layers.py 2011-08-19 13:58:57 +0000 |
188 | +++ lib/canonical/testing/layers.py 2011-09-07 23:29:27 +0000 |
189 | @@ -1555,11 +1555,11 @@ |
190 | @classmethod |
191 | @profiled |
192 | def switchDbUser(cls, dbuser): |
193 | - LaunchpadZopelessLayer.alterConnection(dbuser=dbuser) |
194 | + LaunchpadZopelessLayer._alterConnection(dbuser=dbuser) |
195 | |
196 | @classmethod |
197 | @profiled |
198 | - def alterConnection(cls, **kw): |
199 | + def _alterConnection(cls, **kw): |
200 | """Reset the connection, and reopen the connection by calling |
201 | initZopeless with the given keyword arguments. |
202 | """ |
203 | |
204 | === modified file 'lib/lp/soyuz/scripts/tests/test_queue.py' |
205 | --- lib/lp/soyuz/scripts/tests/test_queue.py 2011-08-01 05:25:59 +0000 |
206 | +++ lib/lp/soyuz/scripts/tests/test_queue.py 2011-09-07 23:29:27 +0000 |
207 | @@ -18,7 +18,6 @@ |
208 | from zope.security.proxy import removeSecurityProxy |
209 | |
210 | from canonical.config import config |
211 | -from canonical.database.sqlbase import ISOLATION_LEVEL_READ_COMMITTED |
212 | from canonical.launchpad.database.librarian import LibraryFileAlias |
213 | from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet |
214 | from canonical.launchpad.interfaces.lpstorm import IStore |
215 | @@ -78,9 +77,7 @@ |
216 | def setUp(self): |
217 | # Switch database user and set isolation level to READ COMMIITTED |
218 | # to avoid SERIALIZATION exceptions with the Librarian. |
219 | - LaunchpadZopelessLayer.alterConnection( |
220 | - dbuser=self.dbuser, |
221 | - isolation=ISOLATION_LEVEL_READ_COMMITTED) |
222 | + LaunchpadZopelessLayer.switchDbUser(self.dbuser) |
223 | |
224 | def _test_display(self, text): |
225 | """Store output from queue tool for inspection.""" |
226 | |
227 | === modified file 'lib/lp/soyuz/tests/soyuz.py' |
228 | --- lib/lp/soyuz/tests/soyuz.py 2011-06-28 15:04:29 +0000 |
229 | +++ lib/lp/soyuz/tests/soyuz.py 2011-09-07 23:29:27 +0000 |
230 | @@ -30,6 +30,7 @@ |
231 | BinaryPackagePublishingHistory, |
232 | SourcePackagePublishingHistory, |
233 | ) |
234 | +from lp.testing.dbuser import dbuser |
235 | from lp.testing.sampledata import ( |
236 | BUILDD_ADMIN_USERNAME, |
237 | CHROOT_LIBRARYFILEALIAS, |
238 | @@ -153,17 +154,13 @@ |
239 | Store the `FakePackager` object used in the test uploads as `packager` |
240 | so the tests can reuse it if necessary. |
241 | """ |
242 | - self.layer.alterConnection(dbuser=LAUNCHPAD_DBUSER_NAME) |
243 | - |
244 | - fake_chroot = LibraryFileAlias.get(CHROOT_LIBRARYFILEALIAS) |
245 | - ubuntu = getUtility(IDistributionSet).getByName( |
246 | - UBUNTU_DISTRIBUTION_NAME) |
247 | - warty = ubuntu.getSeries(WARTY_DISTROSERIES_NAME) |
248 | - warty[I386_ARCHITECTURE_NAME].addOrUpdateChroot(fake_chroot) |
249 | - |
250 | - self.layer.txn.commit() |
251 | - |
252 | - self.layer.alterConnection(dbuser=self.dbuser) |
253 | + with dbuser(LAUNCHPAD_DBUSER_NAME): |
254 | + fake_chroot = LibraryFileAlias.get(CHROOT_LIBRARYFILEALIAS) |
255 | + ubuntu = getUtility(IDistributionSet).getByName( |
256 | + UBUNTU_DISTRIBUTION_NAME) |
257 | + warty = ubuntu.getSeries(WARTY_DISTROSERIES_NAME) |
258 | + warty[I386_ARCHITECTURE_NAME].addOrUpdateChroot(fake_chroot) |
259 | + |
260 | self.packager = self.uploadTestPackages() |
261 | self.layer.txn.commit() |
262 | |
263 | |
264 | === modified file 'lib/lp/soyuz/tests/test_doc.py' |
265 | --- lib/lp/soyuz/tests/test_doc.py 2011-08-12 19:15:43 +0000 |
266 | +++ lib/lp/soyuz/tests/test_doc.py 2011-09-07 23:29:27 +0000 |
267 | @@ -10,10 +10,7 @@ |
268 | import unittest |
269 | |
270 | from canonical.config import config |
271 | -from canonical.database.sqlbase import ( |
272 | - commit, |
273 | - ISOLATION_LEVEL_READ_COMMITTED, |
274 | - ) |
275 | +from canonical.database.sqlbase import commit |
276 | from canonical.launchpad.ftests import logout |
277 | from canonical.launchpad.testing.pages import PageTestSuite |
278 | from canonical.launchpad.testing.systemdocs import ( |
279 | @@ -63,8 +60,7 @@ |
280 | """Setup the connection for the build master tests.""" |
281 | test_dbuser = config.builddmaster.dbuser |
282 | test.globs['test_dbuser'] = test_dbuser |
283 | - LaunchpadZopelessLayer.alterConnection( |
284 | - dbuser=test_dbuser, isolation=ISOLATION_LEVEL_READ_COMMITTED) |
285 | + LaunchpadZopelessLayer.switchDbUser(test_dbuser) |
286 | setGlobs(test) |
287 | |
288 | |
289 | |
290 | === modified file 'lib/lp/soyuz/tests/test_packagediff.py' |
291 | --- lib/lp/soyuz/tests/test_packagediff.py 2011-01-14 11:02:44 +0000 |
292 | +++ lib/lp/soyuz/tests/test_packagediff.py 2011-09-07 23:29:27 +0000 |
293 | @@ -20,6 +20,7 @@ |
294 | from canonical.testing.layers import LaunchpadZopelessLayer |
295 | from lp.soyuz.enums import PackageDiffStatus |
296 | from lp.soyuz.tests.soyuz import TestPackageDiffsBase |
297 | +from lp.testing.dbuser import dbuser |
298 | |
299 | |
300 | class TestPackageDiffs(TestPackageDiffsBase): |
301 | @@ -58,10 +59,8 @@ |
302 | AND sprf.SourcePackageRelease = spr.id |
303 | AND sprf.libraryfile = lfa.id |
304 | """ % sqlvalues(source.id) |
305 | - self.layer.alterConnection(dbuser='launchpad') |
306 | - result = store.execute(query) |
307 | - self.layer.txn.commit() |
308 | - self.layer.alterConnection(dbuser=self.dbuser) |
309 | + with dbuser('launchpad'): |
310 | + store.execute(query) |
311 | |
312 | def test_packagediff_with_expired_and_deleted_lfas(self): |
313 | # Test the case where files required for the diff are expired *and* |
314 | |
315 | === removed file 'utilities/check-sampledata.py' |
316 | --- utilities/check-sampledata.py 2010-04-27 19:48:39 +0000 |
317 | +++ utilities/check-sampledata.py 1970-01-01 00:00:00 +0000 |
318 | @@ -1,364 +0,0 @@ |
319 | -#! /usr/bin/python -S |
320 | -# |
321 | -# Copyright 2009 Canonical Ltd. This software is licensed under the |
322 | -# GNU Affero General Public License version 3 (see the file LICENSE). |
323 | - |
324 | -""" |
325 | -check-sampledata.py - Perform various checks on Sample Data |
326 | - |
327 | -= Launchpad Sample Data Consistency Checks = |
328 | - |
329 | -XXX flacoste 2007/03/08 Once all problems exposed by this script are solved, |
330 | -it should be integrated to our automated test suite. |
331 | - |
332 | -This script verify that all objects in sample data provides the interfaces |
333 | -they are supposed to. It also makes sure that the object pass its schema |
334 | -validation. |
335 | - |
336 | -Finally, it can also be used to report about sample data lacking in breadth. |
337 | - |
338 | -""" |
339 | - |
340 | -__metatype__ = type |
341 | - |
342 | -import _pythonpath |
343 | - |
344 | -import inspect |
345 | -from optparse import OptionParser |
346 | -import re |
347 | -from textwrap import dedent |
348 | - |
349 | -from psycopg2 import ProgrammingError |
350 | - |
351 | -from zope.interface import providedBy |
352 | -from zope.interface.exceptions import ( |
353 | - BrokenImplementation, BrokenMethodImplementation) |
354 | -from zope.interface.verify import verifyObject |
355 | -from zope.schema.interfaces import IField, ValidationError |
356 | - |
357 | -from canonical.database.sqlbase import SQLBase |
358 | -import canonical.launchpad.database |
359 | -from canonical.lp import initZopeless |
360 | -from canonical.launchpad.scripts import execute_zcml_for_scripts |
361 | - |
362 | - |
363 | -def get_class_name(cls): |
364 | - """Return the class name without its package prefix.""" |
365 | - return cls.__name__.split('.')[-1] |
366 | - |
367 | - |
368 | -def error_msg(error): |
369 | - """Convert an exception to a proper error. |
370 | - |
371 | - It make sure that the exception type is in the message and takes care |
372 | - of possible unicode conversion error. |
373 | - """ |
374 | - try: |
375 | - return "%s: %s" % (get_class_name(error.__class__), str(error)) |
376 | - except UnicodeEncodeError: |
377 | - return "UnicodeEncodeError in str(%s)" % error.__class__.__name__ |
378 | - |
379 | - |
380 | -class SampleDataVerification: |
381 | - """Runs various checks on sample data and report about them.""" |
382 | - |
383 | - def __init__(self, dbname="launchpad_ftest_template", dbuser="launchpad", |
384 | - table_filter=None, min_rows=10, only_summary=False): |
385 | - """Initialize the verification object. |
386 | - |
387 | - :param dbname: The database which contains the sample data to check. |
388 | - :param dbuser: The user to connect as. |
389 | - """ |
390 | - self.txn = initZopeless(dbname=dbname, dbuser=dbuser) |
391 | - execute_zcml_for_scripts() |
392 | - self.classes_with_error = {} |
393 | - self.class_rows = {} |
394 | - self.table_filter = table_filter |
395 | - self.min_rows = min_rows |
396 | - self.only_summary = only_summary |
397 | - |
398 | - def findSQLBaseClasses(self): |
399 | - """Return an iterator over the classes in canonical.launchpad.database |
400 | - that extends SQLBase. |
401 | - """ |
402 | - if self.table_filter: |
403 | - include_only_re = re.compile(self.table_filter) |
404 | - for class_name in dir(canonical.launchpad.database): |
405 | - if self.table_filter and not include_only_re.search(class_name): |
406 | - continue |
407 | - cls = getattr(canonical.launchpad.database, class_name) |
408 | - if inspect.isclass(cls) and issubclass(cls, SQLBase): |
409 | - yield cls |
410 | - |
411 | - def fetchTableRowsCount(self): |
412 | - """Fetch the number of rows of each tables. |
413 | - |
414 | - The count are stored in the table_rows_count attribute. |
415 | - """ |
416 | - self.table_rows_count = {} |
417 | - for cls in self.findSQLBaseClasses(): |
418 | - class_name = get_class_name(cls) |
419 | - try: |
420 | - self.table_rows_count[class_name] = cls.select().count() |
421 | - except ProgrammingError, error: |
422 | - self.classes_with_error[class_name] = str(error) |
423 | - # Transaction is borked, start another one. |
424 | - self.txn.begin() |
425 | - |
426 | - def checkSampleDataInterfaces(self): |
427 | - """Check that all sample data objects complies with the interfaces it |
428 | - declares. |
429 | - """ |
430 | - self.validation_errors = {} |
431 | - self.broken_instances= {} |
432 | - for cls in self.findSQLBaseClasses(): |
433 | - class_name = get_class_name(cls) |
434 | - if class_name in self.classes_with_error: |
435 | - continue |
436 | - try: |
437 | - for object in cls.select(): |
438 | - self.checkObjectInterfaces(object) |
439 | - self.validateObjectSchemas(object) |
440 | - except ProgrammingError, error: |
441 | - self.classes_with_error[get_class_name(cls)] = str(error) |
442 | - # Transaction is borked, start another one. |
443 | - self.txn.begin() |
444 | - |
445 | - def checkObjectInterfaces(self, object): |
446 | - """Check that object provides every attributes in its declared interfaces. |
447 | - |
448 | - Collect errors in broken_instances dictionary attribute. |
449 | - """ |
450 | - for interface in providedBy(object): |
451 | - interface_name = get_class_name(interface) |
452 | - try: |
453 | - result = verifyObject(interface, object) |
454 | - except BrokenImplementation, error: |
455 | - self.setInterfaceError( |
456 | - interface, object, "missing attribute %s" % error.name) |
457 | - except BrokenMethodImplementation, error: |
458 | - self.setInterfaceError( |
459 | - interface, object, |
460 | - "invalid method %s: %s" % (error.method, error.mess)) |
461 | - |
462 | - def setInterfaceError(self, interface, object, error_msg): |
463 | - """Store an error about an interface in the broken_instances dictionary |
464 | - |
465 | - The errors data structure looks like: |
466 | - |
467 | - {interface: { |
468 | - error_msg: { |
469 | - class_name: [instance_id...]}}} |
470 | - """ |
471 | - interface_errors = self.broken_instances.setdefault( |
472 | - get_class_name(interface), {}) |
473 | - classes_with_error = interface_errors.setdefault(error_msg, {}) |
474 | - object_ids_with_error = classes_with_error.setdefault( |
475 | - get_class_name(object.__class__), []) |
476 | - object_ids_with_error.append(object.id) |
477 | - |
478 | - def validateObjectSchemas(self, object): |
479 | - """Check that object validates with the schemas it says it provides. |
480 | - |
481 | - Collect errors in validation_errors. Data structure format is |
482 | - {schema: |
483 | - [[class_name, object_id, |
484 | - [(field, error), ...]], |
485 | - ...]} |
486 | - """ |
487 | - for schema in providedBy(object): |
488 | - field_errors = [] |
489 | - for name in schema.names(all=True): |
490 | - description = schema[name] |
491 | - if not IField.providedBy(description): |
492 | - continue |
493 | - try: |
494 | - value = getattr(object, name) |
495 | - except AttributeError: |
496 | - # This is an already reported verifyObject failures. |
497 | - continue |
498 | - try: |
499 | - description.validate(value) |
500 | - except ValidationError, error: |
501 | - field_errors.append((name, error_msg(error))) |
502 | - except (KeyboardInterrupt, SystemExit): |
503 | - # We should never catch KeyboardInterrupt or SystemExit. |
504 | - raise |
505 | - except ProgrammingError, error: |
506 | - field_errors.append((name, error_msg(error))) |
507 | - # We need to restart the transaction after these errors. |
508 | - self.txn.begin() |
509 | - except Exception, error: |
510 | - # Exception usually indicates a deeper problem with |
511 | - # the interface declaration or the validation code, than |
512 | - # the expected ValidationError. |
513 | - field_errors.append((name, error_msg(error))) |
514 | - if field_errors: |
515 | - schema_errors= self.validation_errors.setdefault( |
516 | - get_class_name(schema), []) |
517 | - schema_errors.append([ |
518 | - get_class_name(object.__class__), object.id, |
519 | - field_errors]) |
520 | - |
521 | - def getShortTables(self): |
522 | - """Return a list of tables which have less rows than self.min_rows. |
523 | - |
524 | - :return: [(table, rows_count)...] |
525 | - """ |
526 | - return [ |
527 | - (table, rows_count) |
528 | - for table, rows_count in self.table_rows_count.items() |
529 | - if rows_count < self.min_rows] |
530 | - |
531 | - def reportShortTables(self): |
532 | - """Report about tables with less than self.min_rows.""" |
533 | - short_tables = self.getShortTables() |
534 | - if not short_tables: |
535 | - print """All tables have more than %d rows!!!""" % self.min_rows |
536 | - return |
537 | - |
538 | - print dedent("""\ |
539 | - %d Tables with less than %d rows |
540 | - --------------------------------""" % ( |
541 | - len(short_tables), self.min_rows)) |
542 | - for table, rows_count in sorted(short_tables): |
543 | - print "%-20s: %2d" % (table, rows_count) |
544 | - |
545 | - def reportErrors(self): |
546 | - """Report about classes with database error. |
547 | - |
548 | - This will usually be classes without a database table. |
549 | - """ |
550 | - if not self.classes_with_error: |
551 | - return |
552 | - print dedent("""\ |
553 | - Classes with database errors |
554 | - ----------------------------""") |
555 | - for class_name, error_msg in sorted(self.classes_with_error.items()): |
556 | - print "%-20s %s" % (class_name, error_msg) |
557 | - |
558 | - def reportInterfaceErrors(self): |
559 | - """Report objects failing the verifyObject and schema validation.""" |
560 | - if not self.broken_instances: |
561 | - print "All sample data comply with its provided interfaces!!!" |
562 | - return |
563 | - print dedent("""\ |
564 | - %d Interfaces with broken instances |
565 | - -----------------------------------""" % len( |
566 | - self.broken_instances)) |
567 | - for interface, errors in sorted( |
568 | - self.broken_instances.items()): |
569 | - print "%-20s:" % interface |
570 | - for error_msg, classes_with_error in sorted(errors.items()): |
571 | - print " %s:" % error_msg |
572 | - for class_name, object_ids in sorted( |
573 | - classes_with_error.items()): |
574 | - print " %s: %s" % ( |
575 | - class_name, ", ".join([ |
576 | - str(id) for id in sorted(object_ids)])) |
577 | - |
578 | - def reportValidationErrors(self): |
579 | - """Report object that fails their validation.""" |
580 | - if not self.validation_errors: |
581 | - print "All sample data pass validation!!!" |
582 | - return |
583 | - |
584 | - print dedent("""\ |
585 | - %d Schemas with instances failing validation |
586 | - --------------------------------------------""" % len( |
587 | - self.validation_errors)) |
588 | - for schema, instances in sorted(self.validation_errors.items()): |
589 | - print "%-20s (%d objects with errors):" % (schema, len(instances)) |
590 | - for class_name, object_id, errors in sorted(instances): |
591 | - print " <%s %s> (%d errors):" % ( |
592 | - class_name, object_id, len(errors)) |
593 | - for field, error in sorted(errors): |
594 | - print " %s: %s" % (field, error) |
595 | - |
596 | - def reportSummary(self): |
597 | - """Only report the name of the classes with errors.""" |
598 | - |
599 | - short_tables = dict(self.getShortTables()) |
600 | - |
601 | - # Compute number of implementation error by classes. |
602 | - verify_errors_count = {} |
603 | - for interface_errors in self.broken_instances.values(): |
604 | - for broken_classes in interface_errors.values(): |
605 | - for class_name in broken_classes.keys(): |
606 | - verify_errors_count.setdefault(class_name, 0) |
607 | - verify_errors_count[class_name] += 1 |
608 | - |
609 | - # Compute number of instances with validation error. |
610 | - validation_errors_count = {} |
611 | - for instances in self.validation_errors.values(): |
612 | - for class_name, object_id, errors in instances: |
613 | - validation_errors_count.setdefault(class_name, 0) |
614 | - validation_errors_count[class_name] += 1 |
615 | - |
616 | - classes_with_errors = set(short_tables.keys()) |
617 | - classes_with_errors.update(verify_errors_count.keys()) |
618 | - classes_with_errors.update(validation_errors_count.keys()) |
619 | - |
620 | - print dedent("""\ |
621 | - %d Classes with errors: |
622 | - -----------------------""" % len(classes_with_errors)) |
623 | - for class_name in sorted(classes_with_errors): |
624 | - errors = [] |
625 | - if class_name in short_tables: |
626 | - errors.append('%d rows' % short_tables[class_name]) |
627 | - if class_name in verify_errors_count: |
628 | - errors.append( |
629 | - '%d verify errors' % verify_errors_count[class_name]) |
630 | - if class_name in validation_errors_count: |
631 | - errors.append( |
632 | - '%d validation errors' % |
633 | - validation_errors_count[class_name]) |
634 | - print "%s: %s" % (class_name, ", ".join(errors)) |
635 | - |
636 | - def run(self): |
637 | - """Check and report on sample data.""" |
638 | - self.fetchTableRowsCount() |
639 | - self.checkSampleDataInterfaces() |
640 | - print dedent("""\ |
641 | - Verified %d content classes. |
642 | - ============================ |
643 | - """ % len(self.table_rows_count)) |
644 | - if self.only_summary: |
645 | - self.reportSummary() |
646 | - else: |
647 | - self.reportShortTables() |
648 | |
649 | - self.reportInterfaceErrors() |
650 | |
651 | - self.reportValidationErrors() |
652 | |
653 | - self.reportErrors() |
654 | - self.txn.abort() |
655 | - |
656 | - |
657 | -if __name__ == '__main__': |
658 | - parser = OptionParser() |
659 | - parser.add_option('-d', '--database', action="store", type="string", |
660 | - default="launchpad_ftest_template", |
661 | - help="Database to connect to for testing.") |
662 | - parser.add_option('-u', '--user', action="store", type="string", |
663 | - default="launchpad", |
664 | - help="Username to connect with.") |
665 | - parser.add_option('-i', '--table-filter', dest="table_filter", |
666 | - action="store", type="string", default=None, |
667 | - help="Limit classes to test using a regular expression.") |
668 | - parser.add_option('-m', '--min-rows', dest="min_rows", |
669 | - action="store", type="int", default=10, |
670 | - help="Minimum number of rows a table is expected to have.") |
671 | - parser.add_option('-s', '--summary', |
672 | - action='store_true', dest="summary", default=False, |
673 | - help=( |
674 | - "Only report the name of the classes with " |
675 | - "validation errors.")) |
676 | - options, arguments = parser.parse_args() |
677 | - SampleDataVerification( |
678 | - dbname=options.database, |
679 | - dbuser=options.user, |
680 | - table_filter=options.table_filter, |
681 | - min_rows=options.min_rows, |
682 | - only_summary=options.summary).run() |
Looks good. I didn't know about the dbuser() context manager - that is certainly a nicer way of spelling things.