Merge lp:~allenap/maas/retry-integrity-errors into lp:~maas-committers/maas/trunk
- retry-integrity-errors
- Merge into trunk
Proposed by
Gavin Panella
Status: | Merged |
---|---|
Approved by: | Gavin Panella |
Approved revision: | no longer in the source branch. |
Merged at revision: | 5136 |
Proposed branch: | lp:~allenap/maas/retry-integrity-errors |
Merge into: | lp:~maas-committers/maas/trunk |
Diff against target: |
667 lines (+400/-27) 3 files modified
src/maasserver/testing/testcase.py (+162/-10) src/maasserver/utils/orm.py (+93/-7) src/maasserver/utils/tests/test_orm.py (+145/-10) |
To merge this branch: | bzr merge lp:~allenap/maas/retry-integrity-errors |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Blake Rouse (community) | Approve | ||
Review via email: mp+297673@code.launchpad.net |
Commit message
Retry transactions when they fail with unique violation errors.
This also makes explicit retrying via request_
Description of the change
To post a comment you must log in.
Revision history for this message
Gavin Panella (allenap) wrote : | # |
Thanks!
Revision history for this message
Gavin Panella (allenap) wrote : | # |
Because of the potential for fallout at this late time in the release cycle I'm running this through CI before landing.
Revision history for this message
Gavin Panella (allenap) wrote : | # |
All good in CI.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'src/maasserver/testing/testcase.py' |
2 | --- src/maasserver/testing/testcase.py 2016-05-24 12:51:54 +0000 |
3 | +++ src/maasserver/testing/testcase.py 2016-06-17 14:52:22 +0000 |
4 | @@ -9,9 +9,10 @@ |
5 | 'SeleniumTestCase', |
6 | 'SerializationFailureTestCase', |
7 | 'TestWithoutCrochetMixin', |
8 | + 'UniqueViolationTestCase', |
9 | ] |
10 | |
11 | -from contextlib import closing |
12 | +from itertools import count |
13 | import socketserver |
14 | import sys |
15 | import threading |
16 | @@ -28,13 +29,19 @@ |
17 | connection, |
18 | transaction, |
19 | ) |
20 | -from django.db.utils import OperationalError |
21 | +from django.db.utils import ( |
22 | + IntegrityError, |
23 | + OperationalError, |
24 | +) |
25 | from fixtures import Fixture |
26 | from maasserver.fields import register_mac_type |
27 | from maasserver.testing.factory import factory |
28 | from maasserver.testing.orm import PostCommitHooksTestMixin |
29 | from maasserver.testing.testclient import MAASSensibleClient |
30 | -from maasserver.utils.orm import is_serialization_failure |
31 | +from maasserver.utils.orm import ( |
32 | + is_serialization_failure, |
33 | + is_unique_violation, |
34 | +) |
35 | from maastesting.djangotestcase import ( |
36 | DjangoTestCase, |
37 | DjangoTransactionTestCase, |
38 | @@ -234,11 +241,11 @@ |
39 | DjangoTransactionTestCase, PostCommitHooksTestMixin): |
40 | |
41 | def create_stest_table(self): |
42 | - with closing(connection.cursor()) as cursor: |
43 | + with connection.cursor() as cursor: |
44 | cursor.execute("CREATE TABLE IF NOT EXISTS stest (a INTEGER)") |
45 | |
46 | def drop_stest_table(self): |
47 | - with closing(connection.cursor()) as cursor: |
48 | + with connection.cursor() as cursor: |
49 | cursor.execute("DROP TABLE IF EXISTS stest") |
50 | |
51 | def setUp(self): |
52 | @@ -247,7 +254,7 @@ |
53 | # Put something into the stest table upon which to trigger a |
54 | # serialization failure. |
55 | with transaction.atomic(): |
56 | - with closing(connection.cursor()) as cursor: |
57 | + with connection.cursor() as cursor: |
58 | cursor.execute("INSERT INTO stest VALUES (1)") |
59 | |
60 | def tearDown(self): |
61 | @@ -258,7 +265,7 @@ |
62 | """Trigger an honest, from the database, serialization failure.""" |
63 | # Helper to switch the transaction to SERIALIZABLE. |
64 | def set_serializable(): |
65 | - with closing(connection.cursor()) as cursor: |
66 | + with connection.cursor() as cursor: |
67 | cursor.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE") |
68 | |
69 | # Perform a conflicting update. This must run in a separate thread. It |
70 | @@ -269,7 +276,7 @@ |
71 | def do_conflicting_update(): |
72 | try: |
73 | with transaction.atomic(): |
74 | - with closing(connection.cursor()) as cursor: |
75 | + with connection.cursor() as cursor: |
76 | cursor.execute("UPDATE stest SET a = 2") |
77 | finally: |
78 | close_old_connections() |
79 | @@ -278,7 +285,7 @@ |
80 | # Fetch something first. This ensures that we're inside the |
81 | # transaction, and that the database has a reference point for |
82 | # calculating serialization failures. |
83 | - with closing(connection.cursor()) as cursor: |
84 | + with connection.cursor() as cursor: |
85 | cursor.execute("SELECT * FROM stest") |
86 | cursor.fetchall() |
87 | |
88 | @@ -290,7 +297,7 @@ |
89 | # Updating the same rows as do_conflicting_update() did will |
90 | # trigger a serialization failure. We have to check the __cause__ |
91 | # to confirm the failure type as reported by PostgreSQL. |
92 | - with closing(connection.cursor()) as cursor: |
93 | + with connection.cursor() as cursor: |
94 | cursor.execute("UPDATE stest SET a = 4") |
95 | |
96 | if connection.in_atomic_block: |
97 | @@ -312,3 +319,148 @@ |
98 | return sys.exc_info() |
99 | else: |
100 | raise |
101 | + |
102 | + |
103 | +class UniqueViolationTestCase( |
104 | + DjangoTransactionTestCase, PostCommitHooksTestMixin): |
105 | + |
106 | + def create_uvtest_table(self): |
107 | + with connection.cursor() as cursor: |
108 | + cursor.execute("DROP TABLE IF EXISTS uvtest") |
109 | + cursor.execute("CREATE TABLE uvtest (a INTEGER PRIMARY KEY)") |
110 | + |
111 | + def drop_uvtest_table(self): |
112 | + with connection.cursor() as cursor: |
113 | + cursor.execute("DROP TABLE IF EXISTS uvtest") |
114 | + |
115 | + def setUp(self): |
116 | + super(UniqueViolationTestCase, self).setUp() |
117 | + self.conflicting_values = count(1) |
118 | + self.create_uvtest_table() |
119 | + |
120 | + def tearDown(self): |
121 | + super(UniqueViolationTestCase, self).tearDown() |
122 | + self.drop_uvtest_table() |
123 | + |
124 | + def cause_unique_violation(self): |
125 | + """Trigger an honest, from the database, unique violation. |
126 | + |
127 | + This may appear needlessly elaborate, but it's for a good reason. |
128 | + Indexes in PostgreSQL are a bit weird; they don't fully support MVCC |
129 | + so it's possible for situations like the following: |
130 | + |
131 | + CREATE TABLE foo (id SERIAL PRIMARY KEY); |
132 | + -- Session A: |
133 | + BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; |
134 | + INSERT INTO foo (id) VALUES (1); |
135 | + -- Session B: |
136 | + BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; |
137 | + SELECT id FROM foo; -- Nothing. |
138 | + INSERT INTO foo (id) VALUES (1); -- Hangs. |
139 | + -- Session A: |
140 | + COMMIT; |
141 | + -- Session B: |
142 | + ERROR: duplicate key value violates unique constraint "..." |
143 | + DETAIL: Key (id)=(1) already exists. |
144 | + |
145 | + Two things to note: |
146 | + |
147 | + 1. Session B hangs when there's a potential conflict on id's index. |
148 | + |
149 | + 2. Session B fails with a duplicate key error. |
150 | + |
151 | + Both differ from expectations: |
152 | + |
153 | + 1. I would expect the transaction to continue optimistically and |
154 | + only fail if session A commits. |
155 | + |
156 | + 2. I would expect a serialisation failure instead. |
157 | + |
158 | + This method jumps through hoops to reproduce the situation above so |
159 | + that we're testing against PostgreSQL's exact behaviour as of today, |
160 | + not the behaviour that we observed at a single moment in time. |
161 | + PostgreSQL may change its behaviour in later versions and this test |
162 | + ought to tell us about it. |
163 | + |
164 | + """ |
165 | + # Helper to switch the transaction to REPEATABLE READ. |
166 | + def set_repeatable_read(): |
167 | + with connection.cursor() as cursor: |
168 | + cursor.execute( |
169 | + "SET TRANSACTION ISOLATION LEVEL " |
170 | + "REPEATABLE READ") |
171 | + |
172 | + # Both threads / database sessions will attempt to insert this. |
173 | + conflicting_value = next(self.conflicting_values) |
174 | + |
175 | + # Perform a conflicting insert. This must run in a separate thread. It |
176 | + # also must begin after the beginning of the transaction in which we |
177 | + # will trigger a unique violation AND commit before that other |
178 | + # transaction commits. This doesn't need to run with any special |
179 | + # isolation; it just needs to be in a transaction. |
180 | + def do_conflicting_insert(): |
181 | + try: |
182 | + with transaction.atomic(): |
183 | + with connection.cursor() as cursor: |
184 | + cursor.execute( |
185 | + "INSERT INTO uvtest VALUES (%s)", |
186 | + [conflicting_value]) |
187 | + finally: |
188 | + close_old_connections() |
189 | + |
190 | + def trigger_unique_violation(): |
191 | + # Fetch something first. This ensures that we're inside the |
192 | + # transaction, and so the database has a reference point for |
193 | + # repeatable reads. |
194 | + with connection.cursor() as cursor: |
195 | + cursor.execute( |
196 | + "SELECT 1 FROM uvtest WHERE a = %s", |
197 | + [conflicting_value]) |
198 | + self.assertIsNone(cursor.fetchone(), ( |
199 | + "We've seen through PostgreSQL impenetrable transaction " |
200 | + "isolation — or so we once thought — to witness a " |
201 | + "conflicting value from another database session. " |
202 | + "Needless to say, this requires investigation.")) |
203 | + |
204 | + # Run do_conflicting_insert() in a separate thread and wait for it |
205 | + # to commit and return. |
206 | + thread = threading.Thread(target=do_conflicting_insert) |
207 | + thread.start() |
208 | + thread.join() |
209 | + |
210 | + # Still no sign of that conflicting value from here. |
211 | + with connection.cursor() as cursor: |
212 | + cursor.execute( |
213 | + "SELECT 1 FROM uvtest WHERE a = %s", |
214 | + [conflicting_value]) |
215 | + self.assertIsNone(cursor.fetchone(), ( |
216 | + "PostgreSQL, once thought of highly in transactional " |
217 | + "circles, has dropped its kimono and disgraced itself " |
218 | + "with its wanton exhibition of conflicting values from " |
219 | + "another's session.")) |
220 | + |
221 | + # Inserting the same row will trigger a unique violation. |
222 | + with connection.cursor() as cursor: |
223 | + cursor.execute( |
224 | + "INSERT INTO uvtest VALUES (%s)", |
225 | + [conflicting_value]) |
226 | + |
227 | + if connection.in_atomic_block: |
228 | + # We're already in a transaction. |
229 | + set_repeatable_read() |
230 | + trigger_unique_violation() |
231 | + else: |
232 | + # Start a transaction in this thread. |
233 | + with transaction.atomic(): |
234 | + set_repeatable_read() |
235 | + trigger_unique_violation() |
236 | + |
237 | + def capture_unique_violation(self): |
238 | + """Trigger a unique violation, return its ``exc_info`` tuple.""" |
239 | + try: |
240 | + self.cause_unique_violation() |
241 | + except IntegrityError as e: |
242 | + if is_unique_violation(e): |
243 | + return sys.exc_info() |
244 | + else: |
245 | + raise |
246 | |
247 | === modified file 'src/maasserver/utils/orm.py' |
248 | --- src/maasserver/utils/orm.py 2016-03-28 13:54:47 +0000 |
249 | +++ src/maasserver/utils/orm.py 2016-06-17 14:52:22 +0000 |
250 | @@ -16,8 +16,10 @@ |
251 | 'is_deadlock_failure', |
252 | 'is_retryable_failure', |
253 | 'is_serialization_failure', |
254 | + 'is_unique_violation', |
255 | 'make_deadlock_failure', |
256 | 'make_serialization_failure', |
257 | + 'make_unique_violation', |
258 | 'post_commit', |
259 | 'post_commit_do', |
260 | 'psql_array', |
261 | @@ -54,7 +56,11 @@ |
262 | ) |
263 | from django.db.models import Q |
264 | from django.db.transaction import TransactionManagementError |
265 | -from django.db.utils import OperationalError |
266 | +from django.db.utils import ( |
267 | + DatabaseError, |
268 | + IntegrityError, |
269 | + OperationalError, |
270 | +) |
271 | from django.http import Http404 |
272 | from maasserver.exceptions import ( |
273 | MAASAPIBadRequest, |
274 | @@ -72,6 +78,7 @@ |
275 | from psycopg2.errorcodes import ( |
276 | DEADLOCK_DETECTED, |
277 | SERIALIZATION_FAILURE, |
278 | + UNIQUE_VIOLATION, |
279 | ) |
280 | from twisted.internet.defer import Deferred |
281 | |
282 | @@ -273,6 +280,68 @@ |
283 | return get_psycopg2_deadlock_exception(exception) is not None |
284 | |
285 | |
286 | +def get_psycopg2_unique_violation_exception(exception): |
287 | + """Return the root-cause if `exception` is a unique violation. |
288 | + |
289 | + PostgreSQL sets a specific error code, "23505", when a transaction breaks |
290 | + because of a unique violation. |
291 | + |
292 | + :return: The underlying `psycopg2.Error` if it's a unique violation, or |
293 | + `None` if there isn't one. |
294 | + """ |
295 | + exception = get_psycopg2_exception(exception) |
296 | + if exception is None: |
297 | + return None |
298 | + elif exception.pgcode == UNIQUE_VIOLATION: |
299 | + return exception |
300 | + else: |
301 | + return None |
302 | + |
303 | + |
304 | +def is_unique_violation(exception): |
305 | + """Does `exception` represent a unique violation? |
306 | + |
307 | + PostgreSQL sets a specific error code, "23505", when a transaction breaks |
308 | + because of a unique violation. |
309 | + """ |
310 | + return get_psycopg2_unique_violation_exception(exception) is not None |
311 | + |
312 | + |
313 | +class UniqueViolation(psycopg2.IntegrityError): |
314 | + """Explicit serialization failure. |
315 | + |
316 | + A real unique violation, arising out of psycopg2 (and thus signalled from |
317 | + the database) would *NOT* be an instance of this class. However, it is not |
318 | + obvious how to create a `psycopg2.IntegrityError` with ``pgcode`` set to |
319 | + `UNIQUE_VIOLATION` without subclassing. I suspect only the C interface can |
320 | + do that. |
321 | + """ |
322 | + pgcode = UNIQUE_VIOLATION |
323 | + |
324 | + |
325 | +def make_unique_violation(): |
326 | + """Make a serialization exception. |
327 | + |
328 | + Artificially construct an exception that resembles what Django's ORM would |
329 | + raise when PostgreSQL fails a transaction because of a unique violation. |
330 | + |
331 | + :returns: an instance of :py:class:`IntegrityError` that will pass the |
332 | + `is_unique_violation` predicate. |
333 | + """ |
334 | + exception = IntegrityError() |
335 | + exception.__cause__ = UniqueViolation() |
336 | + assert is_unique_violation(exception) |
337 | + return exception |
338 | + |
339 | + |
340 | +class RetryTransaction(BaseException): |
341 | + """An explicit request that the transaction be retried.""" |
342 | + |
343 | + |
344 | +class TooManyRetries(Exception): |
345 | + """A transaction retry has been requested too many times.""" |
346 | + |
347 | + |
348 | def request_transaction_retry(): |
349 | """Raise a serialization exception. |
350 | |
351 | @@ -280,15 +349,24 @@ |
352 | this, and then retrying the transaction, though it may choose to re-raise |
353 | the error if too many retries have already been attempted. |
354 | |
355 | - :raises OperationalError: |
356 | + :raise RetryTransaction: |
357 | """ |
358 | - raise make_serialization_failure() |
359 | + raise RetryTransaction() |
360 | |
361 | |
362 | def is_retryable_failure(exception): |
363 | - """Does `exception` represent a serialization or deadlock failure?""" |
364 | + """Does `exception` represent a retryable failure? |
365 | + |
366 | + This does NOT include requested retries, i.e. `RetryTransaction`. |
367 | + |
368 | + :param exception: An instance of :class:`DatabaseError` or one of its |
369 | + subclasses. |
370 | + """ |
371 | return ( |
372 | - is_serialization_failure(exception) or is_deadlock_failure(exception)) |
373 | + is_serialization_failure(exception) or |
374 | + is_deadlock_failure(exception) or |
375 | + is_unique_violation(exception) |
376 | + ) |
377 | |
378 | |
379 | def gen_retry_intervals(base=0.01, rate=2.5, maximum=10.0): |
380 | @@ -341,14 +419,22 @@ |
381 | for _ in range(9): |
382 | try: |
383 | return func(*args, **kwargs) |
384 | - except OperationalError as error: |
385 | + except RetryTransaction: |
386 | + reset() # Which may do nothing. |
387 | + sleep(next(intervals)) |
388 | + except DatabaseError as error: |
389 | if is_retryable_failure(error): |
390 | reset() # Which may do nothing. |
391 | sleep(next(intervals)) |
392 | else: |
393 | raise |
394 | else: |
395 | - return func(*args, **kwargs) |
396 | + try: |
397 | + return func(*args, **kwargs) |
398 | + except RetryTransaction: |
399 | + raise TooManyRetries( |
400 | + "This transaction has already been attempted " |
401 | + "multiple times; giving up.") |
402 | return retrier |
403 | |
404 | |
405 | |
406 | === modified file 'src/maasserver/utils/tests/test_orm.py' |
407 | --- src/maasserver/utils/tests/test_orm.py 2016-05-12 19:07:37 +0000 |
408 | +++ src/maasserver/utils/tests/test_orm.py 2016-06-17 14:52:22 +0000 |
409 | @@ -26,12 +26,16 @@ |
410 | ) |
411 | from django.db.backends.base.base import BaseDatabaseWrapper |
412 | from django.db.transaction import TransactionManagementError |
413 | -from django.db.utils import OperationalError |
414 | +from django.db.utils import ( |
415 | + IntegrityError, |
416 | + OperationalError, |
417 | +) |
418 | from maasserver.models import Node |
419 | from maasserver.testing.testcase import ( |
420 | MAASServerTestCase, |
421 | MAASTransactionServerTestCase, |
422 | SerializationFailureTestCase, |
423 | + UniqueViolationTestCase, |
424 | ) |
425 | from maasserver.utils import orm |
426 | from maasserver.utils.orm import ( |
427 | @@ -46,11 +50,12 @@ |
428 | get_psycopg2_deadlock_exception, |
429 | get_psycopg2_exception, |
430 | get_psycopg2_serialization_exception, |
431 | + get_psycopg2_unique_violation_exception, |
432 | in_transaction, |
433 | is_deadlock_failure, |
434 | is_retryable_failure, |
435 | is_serialization_failure, |
436 | - make_serialization_failure, |
437 | + is_unique_violation, |
438 | post_commit, |
439 | post_commit_do, |
440 | post_commit_hooks, |
441 | @@ -81,6 +86,7 @@ |
442 | from psycopg2.errorcodes import ( |
443 | DEADLOCK_DETECTED, |
444 | SERIALIZATION_FAILURE, |
445 | + UNIQUE_VIOLATION, |
446 | ) |
447 | from testtools import ExpectedException |
448 | from testtools.matchers import ( |
449 | @@ -221,6 +227,16 @@ |
450 | SERIALIZATION_FAILURE, error.__cause__.pgcode) |
451 | |
452 | |
453 | +class TestUniqueViolation(UniqueViolationTestCase): |
454 | + """Detecting UNIQUE_VIOLATION failures.""" |
455 | + |
456 | + def test_unique_violation_detectable_via_error_cause(self): |
457 | + error = self.assertRaises( |
458 | + IntegrityError, self.cause_unique_violation) |
459 | + self.assertEqual( |
460 | + UNIQUE_VIOLATION, error.__cause__.pgcode) |
461 | + |
462 | + |
463 | class TestGetPsycopg2Exception(MAASTestCase): |
464 | """Tests for `get_psycopg2_exception`.""" |
465 | |
466 | @@ -281,6 +297,25 @@ |
467 | get_psycopg2_deadlock_exception(exception)) |
468 | |
469 | |
470 | +class TestGetPsycopg2UniqueViolationException(MAASTestCase): |
471 | + """Tests for `get_psycopg2_unique_violation_exception`.""" |
472 | + |
473 | + def test__returns_None_for_plain_psycopg2_error(self): |
474 | + exception = psycopg2.Error() |
475 | + self.assertIsNone(get_psycopg2_unique_violation_exception(exception)) |
476 | + |
477 | + def test__returns_None_for_other_error(self): |
478 | + exception = factory.make_exception() |
479 | + self.assertIsNone(get_psycopg2_unique_violation_exception(exception)) |
480 | + |
481 | + def test__returns_psycopg2_error_root_cause(self): |
482 | + exception = Exception() |
483 | + exception.__cause__ = orm.UniqueViolation() |
484 | + self.assertIs( |
485 | + exception.__cause__, |
486 | + get_psycopg2_unique_violation_exception(exception)) |
487 | + |
488 | + |
489 | class TestIsSerializationFailure(SerializationFailureTestCase): |
490 | """Tests relating to MAAS's use of SERIALIZABLE isolation.""" |
491 | |
492 | @@ -340,6 +375,36 @@ |
493 | self.assertFalse(is_deadlock_failure(error)) |
494 | |
495 | |
496 | +class TestIsUniqueViolation(UniqueViolationTestCase): |
497 | + """Tests relating to MAAS's identification of unique violations.""" |
498 | + |
499 | + def test_detects_integrity_error_with_matching_cause(self): |
500 | + error = self.assertRaises( |
501 | + IntegrityError, self.cause_unique_violation) |
502 | + self.assertTrue(is_unique_violation(error)) |
503 | + |
504 | + def test_rejects_integrity_error_without_matching_cause(self): |
505 | + error = IntegrityError() |
506 | + cause = self.patch(error, "__cause__", Exception()) |
507 | + cause.pgcode = factory.make_name("pgcode") |
508 | + self.assertFalse(is_unique_violation(error)) |
509 | + |
510 | + def test_rejects_integrity_error_with_unrelated_cause(self): |
511 | + error = IntegrityError() |
512 | + error.__cause__ = Exception() |
513 | + self.assertFalse(is_unique_violation(error)) |
514 | + |
515 | + def test_rejects_integrity_error_without_cause(self): |
516 | + error = IntegrityError() |
517 | + self.assertFalse(is_unique_violation(error)) |
518 | + |
519 | + def test_rejects_non_integrity_error_with_matching_cause(self): |
520 | + error = factory.make_exception() |
521 | + cause = self.patch(error, "__cause__", Exception()) |
522 | + cause.pgcode = UNIQUE_VIOLATION |
523 | + self.assertFalse(is_unique_violation(error)) |
524 | + |
525 | + |
526 | class TestIsRetryableFailure(MAASTestCase): |
527 | """Tests relating to MAAS's use of catching retryable failures.""" |
528 | |
529 | @@ -351,33 +416,58 @@ |
530 | error = orm.make_deadlock_failure() |
531 | self.assertTrue(is_retryable_failure(error)) |
532 | |
533 | + def test_detects_unique_violation(self): |
534 | + error = orm.make_unique_violation() |
535 | + self.assertTrue(is_retryable_failure(error)) |
536 | + |
537 | def test_rejects_operational_error_without_matching_cause(self): |
538 | error = OperationalError() |
539 | cause = self.patch(error, "__cause__", Exception()) |
540 | cause.pgcode = factory.make_name("pgcode") |
541 | self.assertFalse(is_retryable_failure(error)) |
542 | |
543 | + def test_rejects_integrity_error_without_matching_cause(self): |
544 | + error = IntegrityError() |
545 | + cause = self.patch(error, "__cause__", Exception()) |
546 | + cause.pgcode = factory.make_name("pgcode") |
547 | + self.assertFalse(is_retryable_failure(error)) |
548 | + |
549 | def test_rejects_operational_error_with_unrelated_cause(self): |
550 | error = OperationalError() |
551 | error.__cause__ = Exception() |
552 | self.assertFalse(is_retryable_failure(error)) |
553 | |
554 | + def test_rejects_integrity_error_with_unrelated_cause(self): |
555 | + error = IntegrityError() |
556 | + error.__cause__ = Exception() |
557 | + self.assertFalse(is_retryable_failure(error)) |
558 | + |
559 | def test_rejects_operational_error_without_cause(self): |
560 | error = OperationalError() |
561 | self.assertFalse(is_retryable_failure(error)) |
562 | |
563 | - def test_rejects_non_operational_error_with_cause_serialization(self): |
564 | + def test_rejects_integrity_error_without_cause(self): |
565 | + error = IntegrityError() |
566 | + self.assertFalse(is_retryable_failure(error)) |
567 | + |
568 | + def test_rejects_non_database_error_with_cause_serialization(self): |
569 | error = factory.make_exception() |
570 | cause = self.patch(error, "__cause__", Exception()) |
571 | cause.pgcode = SERIALIZATION_FAILURE |
572 | self.assertFalse(is_retryable_failure(error)) |
573 | |
574 | - def test_rejects_non_operational_error_with_cause_deadlock(self): |
575 | + def test_rejects_non_database_error_with_cause_deadlock(self): |
576 | error = factory.make_exception() |
577 | cause = self.patch(error, "__cause__", Exception()) |
578 | cause.pgcode = DEADLOCK_DETECTED |
579 | self.assertFalse(is_retryable_failure(error)) |
580 | |
581 | + def test_rejects_non_database_error_with_cause_unique_violation(self): |
582 | + error = factory.make_exception() |
583 | + cause = self.patch(error, "__cause__", Exception()) |
584 | + cause.pgcode = UNIQUE_VIOLATION |
585 | + self.assertFalse(is_retryable_failure(error)) |
586 | + |
587 | |
588 | class TestRetryOnRetryableFailure(SerializationFailureTestCase): |
589 | |
590 | @@ -418,6 +508,36 @@ |
591 | self.assertEqual(sentinel.result, function_wrapped()) |
592 | self.assertThat(function, MockCallsMatch(call(), call())) |
593 | |
594 | + def test_retries_on_unique_violation(self): |
595 | + function = self.make_mock_function() |
596 | + function.side_effect = orm.make_unique_violation() |
597 | + function_wrapped = retry_on_retryable_failure(function) |
598 | + self.assertRaises(IntegrityError, function_wrapped) |
599 | + expected_calls = [call()] * 10 |
600 | + self.assertThat(function, MockCallsMatch(*expected_calls)) |
601 | + |
602 | + def test_retries_on_unique_violation_until_successful(self): |
603 | + function = self.make_mock_function() |
604 | + function.side_effect = [orm.make_unique_violation(), sentinel.result] |
605 | + function_wrapped = retry_on_retryable_failure(function) |
606 | + self.assertEqual(sentinel.result, function_wrapped()) |
607 | + self.assertThat(function, MockCallsMatch(call(), call())) |
608 | + |
609 | + def test_retries_on_retry_transaction(self): |
610 | + function = self.make_mock_function() |
611 | + function.side_effect = orm.RetryTransaction() |
612 | + function_wrapped = retry_on_retryable_failure(function) |
613 | + self.assertRaises(orm.TooManyRetries, function_wrapped) |
614 | + expected_calls = [call()] * 10 |
615 | + self.assertThat(function, MockCallsMatch(*expected_calls)) |
616 | + |
617 | + def test_retries_on_retry_transaction_until_successful(self): |
618 | + function = self.make_mock_function() |
619 | + function.side_effect = [orm.RetryTransaction(), sentinel.result] |
620 | + function_wrapped = retry_on_retryable_failure(function) |
621 | + self.assertEqual(sentinel.result, function_wrapped()) |
622 | + self.assertThat(function, MockCallsMatch(call(), call())) |
623 | + |
624 | def test_passes_args_to_wrapped_function(self): |
625 | function = lambda a, b: (a, b) |
626 | function_wrapped = retry_on_retryable_failure(function) |
627 | @@ -450,19 +570,34 @@ |
628 | """Tests for `make_serialization_failure`.""" |
629 | |
630 | def test__makes_a_serialization_failure(self): |
631 | - exception = make_serialization_failure() |
632 | + exception = orm.make_serialization_failure() |
633 | self.assertThat(exception, MatchesPredicate( |
634 | is_serialization_failure, "%r is not a serialization failure.")) |
635 | |
636 | |
637 | +class TestMakeDeadlockFailure(MAASTestCase): |
638 | + """Tests for `make_deadlock_failure`.""" |
639 | + |
640 | + def test__makes_a_deadlock_failure(self): |
641 | + exception = orm.make_deadlock_failure() |
642 | + self.assertThat(exception, MatchesPredicate( |
643 | + is_deadlock_failure, "%r is not a deadlock failure.")) |
644 | + |
645 | + |
646 | +class TestMakeUniqueViolation(MAASTestCase): |
647 | + """Tests for `make_unique_violation`.""" |
648 | + |
649 | + def test__makes_a_unique_violation(self): |
650 | + exception = orm.make_unique_violation() |
651 | + self.assertThat(exception, MatchesPredicate( |
652 | + is_unique_violation, "%r is not a unique violation.")) |
653 | + |
654 | + |
655 | class TestRequestTransactionRetry(MAASTestCase): |
656 | """Tests for `request_transaction_retry`.""" |
657 | |
658 | - def test__raises_a_serialization_failure(self): |
659 | - exception = self.assertRaises( |
660 | - OperationalError, request_transaction_retry) |
661 | - self.assertThat(exception, MatchesPredicate( |
662 | - is_serialization_failure, "%r is not a serialization failure.")) |
663 | + def test__raises_a_retry_transaction_exception(self): |
664 | + self.assertRaises(orm.RetryTransaction, request_transaction_retry) |
665 | |
666 | |
667 | class TestGenRetryIntervals(MAASTestCase): |
Looks really good. The test case for creating the exception is awesome. Just some comments and 1 question.