Merge lp:~allenap/maas/retry-integrity-errors into lp:maas/trunk

Proposed by Gavin Panella on 2016-06-16
Status: Merged
Approved by: Gavin Panella on 2016-06-19
Approved revision: 5129
Merged at revision: 5136
Proposed branch: lp:~allenap/maas/retry-integrity-errors
Merge into: lp:maas/trunk
Diff against target: 667 lines (+400/-27)
3 files modified
src/maasserver/testing/testcase.py (+162/-10)
src/maasserver/utils/orm.py (+93/-7)
src/maasserver/utils/tests/test_orm.py (+145/-10)
To merge this branch: bzr merge lp:~allenap/maas/retry-integrity-errors
Reviewer Review Type Date Requested Status
Blake Rouse 2016-06-16 Approve on 2016-06-17
Review via email: mp+297673@code.launchpad.net

Commit message

Retry transactions when they fail with unique violation errors.

This also makes explicit retrying via request_transaction_retry a separate thing rather than a piggyback on serialization failures.

To post a comment you must log in.
Blake Rouse (blake-rouse) wrote :

Looks really good. The test case for creating the exception is awesome. Just some comments and 1 question.

review: Approve
5128. By Gavin Panella on 2016-06-17

Explain about seeing other's conflicting values in UniqueViolationTestCase.

5129. By Gavin Panella on 2016-06-17

Fix typo.

Gavin Panella (allenap) wrote :

Thanks!

Gavin Panella (allenap) wrote :

Because of the potential for fallout at this late time in the release cycle I'm running this through CI before landing.

Gavin Panella (allenap) wrote :

All good in CI.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'src/maasserver/testing/testcase.py'
2--- src/maasserver/testing/testcase.py 2016-05-24 12:51:54 +0000
3+++ src/maasserver/testing/testcase.py 2016-06-17 14:52:22 +0000
4@@ -9,9 +9,10 @@
5 'SeleniumTestCase',
6 'SerializationFailureTestCase',
7 'TestWithoutCrochetMixin',
8+ 'UniqueViolationTestCase',
9 ]
10
11-from contextlib import closing
12+from itertools import count
13 import socketserver
14 import sys
15 import threading
16@@ -28,13 +29,19 @@
17 connection,
18 transaction,
19 )
20-from django.db.utils import OperationalError
21+from django.db.utils import (
22+ IntegrityError,
23+ OperationalError,
24+)
25 from fixtures import Fixture
26 from maasserver.fields import register_mac_type
27 from maasserver.testing.factory import factory
28 from maasserver.testing.orm import PostCommitHooksTestMixin
29 from maasserver.testing.testclient import MAASSensibleClient
30-from maasserver.utils.orm import is_serialization_failure
31+from maasserver.utils.orm import (
32+ is_serialization_failure,
33+ is_unique_violation,
34+)
35 from maastesting.djangotestcase import (
36 DjangoTestCase,
37 DjangoTransactionTestCase,
38@@ -234,11 +241,11 @@
39 DjangoTransactionTestCase, PostCommitHooksTestMixin):
40
41 def create_stest_table(self):
42- with closing(connection.cursor()) as cursor:
43+ with connection.cursor() as cursor:
44 cursor.execute("CREATE TABLE IF NOT EXISTS stest (a INTEGER)")
45
46 def drop_stest_table(self):
47- with closing(connection.cursor()) as cursor:
48+ with connection.cursor() as cursor:
49 cursor.execute("DROP TABLE IF EXISTS stest")
50
51 def setUp(self):
52@@ -247,7 +254,7 @@
53 # Put something into the stest table upon which to trigger a
54 # serialization failure.
55 with transaction.atomic():
56- with closing(connection.cursor()) as cursor:
57+ with connection.cursor() as cursor:
58 cursor.execute("INSERT INTO stest VALUES (1)")
59
60 def tearDown(self):
61@@ -258,7 +265,7 @@
62 """Trigger an honest, from the database, serialization failure."""
63 # Helper to switch the transaction to SERIALIZABLE.
64 def set_serializable():
65- with closing(connection.cursor()) as cursor:
66+ with connection.cursor() as cursor:
67 cursor.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE")
68
69 # Perform a conflicting update. This must run in a separate thread. It
70@@ -269,7 +276,7 @@
71 def do_conflicting_update():
72 try:
73 with transaction.atomic():
74- with closing(connection.cursor()) as cursor:
75+ with connection.cursor() as cursor:
76 cursor.execute("UPDATE stest SET a = 2")
77 finally:
78 close_old_connections()
79@@ -278,7 +285,7 @@
80 # Fetch something first. This ensures that we're inside the
81 # transaction, and that the database has a reference point for
82 # calculating serialization failures.
83- with closing(connection.cursor()) as cursor:
84+ with connection.cursor() as cursor:
85 cursor.execute("SELECT * FROM stest")
86 cursor.fetchall()
87
88@@ -290,7 +297,7 @@
89 # Updating the same rows as do_conflicting_update() did will
90 # trigger a serialization failure. We have to check the __cause__
91 # to confirm the failure type as reported by PostgreSQL.
92- with closing(connection.cursor()) as cursor:
93+ with connection.cursor() as cursor:
94 cursor.execute("UPDATE stest SET a = 4")
95
96 if connection.in_atomic_block:
97@@ -312,3 +319,148 @@
98 return sys.exc_info()
99 else:
100 raise
101+
102+
103+class UniqueViolationTestCase(
104+ DjangoTransactionTestCase, PostCommitHooksTestMixin):
105+
106+ def create_uvtest_table(self):
107+ with connection.cursor() as cursor:
108+ cursor.execute("DROP TABLE IF EXISTS uvtest")
109+ cursor.execute("CREATE TABLE uvtest (a INTEGER PRIMARY KEY)")
110+
111+ def drop_uvtest_table(self):
112+ with connection.cursor() as cursor:
113+ cursor.execute("DROP TABLE IF EXISTS uvtest")
114+
115+ def setUp(self):
116+ super(UniqueViolationTestCase, self).setUp()
117+ self.conflicting_values = count(1)
118+ self.create_uvtest_table()
119+
120+ def tearDown(self):
121+ super(UniqueViolationTestCase, self).tearDown()
122+ self.drop_uvtest_table()
123+
124+ def cause_unique_violation(self):
125+ """Trigger an honest, from the database, unique violation.
126+
127+ This may appear needlessly elaborate, but it's for a good reason.
128+ Indexes in PostgreSQL are a bit weird; they don't fully support MVCC
129+ so it's possible for situations like the following:
130+
131+ CREATE TABLE foo (id SERIAL PRIMARY KEY);
132+ -- Session A:
133+ BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;
134+ INSERT INTO foo (id) VALUES (1);
135+ -- Session B:
136+ BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;
137+ SELECT id FROM foo; -- Nothing.
138+ INSERT INTO foo (id) VALUES (1); -- Hangs.
139+ -- Session A:
140+ COMMIT;
141+ -- Session B:
142+ ERROR: duplicate key value violates unique constraint "..."
143+ DETAIL: Key (id)=(1) already exists.
144+
145+ Two things to note:
146+
147+ 1. Session B hangs when there's a potential conflict on id's index.
148+
149+ 2. Session B fails with a duplicate key error.
150+
151+ Both differ from expectations:
152+
153+ 1. I would expect the transaction to continue optimistically and
154+ only fail if session A commits.
155+
156+ 2. I would expect a serialisation failure instead.
157+
158+ This method jumps through hoops to reproduce the situation above so
159+ that we're testing against PostgreSQL's exact behaviour as of today,
160+ not the behaviour that we observed at a single moment in time.
161+ PostgreSQL may change its behaviour in later versions and this test
162+ ought to tell us about it.
163+
164+ """
165+ # Helper to switch the transaction to REPEATABLE READ.
166+ def set_repeatable_read():
167+ with connection.cursor() as cursor:
168+ cursor.execute(
169+ "SET TRANSACTION ISOLATION LEVEL "
170+ "REPEATABLE READ")
171+
172+ # Both threads / database sessions will attempt to insert this.
173+ conflicting_value = next(self.conflicting_values)
174+
175+ # Perform a conflicting insert. This must run in a separate thread. It
176+ # also must begin after the beginning of the transaction in which we
177+ # will trigger a unique violation AND commit before that other
178+ # transaction commits. This doesn't need to run with any special
179+ # isolation; it just needs to be in a transaction.
180+ def do_conflicting_insert():
181+ try:
182+ with transaction.atomic():
183+ with connection.cursor() as cursor:
184+ cursor.execute(
185+ "INSERT INTO uvtest VALUES (%s)",
186+ [conflicting_value])
187+ finally:
188+ close_old_connections()
189+
190+ def trigger_unique_violation():
191+ # Fetch something first. This ensures that we're inside the
192+ # transaction, and so the database has a reference point for
193+ # repeatable reads.
194+ with connection.cursor() as cursor:
195+ cursor.execute(
196+ "SELECT 1 FROM uvtest WHERE a = %s",
197+ [conflicting_value])
198+ self.assertIsNone(cursor.fetchone(), (
199+ "We've seen through PostgreSQL impenetrable transaction "
200+ "isolation — or so we once thought — to witness a "
201+ "conflicting value from another database session. "
202+ "Needless to say, this requires investigation."))
203+
204+ # Run do_conflicting_insert() in a separate thread and wait for it
205+ # to commit and return.
206+ thread = threading.Thread(target=do_conflicting_insert)
207+ thread.start()
208+ thread.join()
209+
210+ # Still no sign of that conflicting value from here.
211+ with connection.cursor() as cursor:
212+ cursor.execute(
213+ "SELECT 1 FROM uvtest WHERE a = %s",
214+ [conflicting_value])
215+ self.assertIsNone(cursor.fetchone(), (
216+ "PostgreSQL, once thought of highly in transactional "
217+ "circles, has dropped its kimono and disgraced itself "
218+ "with its wanton exhibition of conflicting values from "
219+ "another's session."))
220+
221+ # Inserting the same row will trigger a unique violation.
222+ with connection.cursor() as cursor:
223+ cursor.execute(
224+ "INSERT INTO uvtest VALUES (%s)",
225+ [conflicting_value])
226+
227+ if connection.in_atomic_block:
228+ # We're already in a transaction.
229+ set_repeatable_read()
230+ trigger_unique_violation()
231+ else:
232+ # Start a transaction in this thread.
233+ with transaction.atomic():
234+ set_repeatable_read()
235+ trigger_unique_violation()
236+
237+ def capture_unique_violation(self):
238+ """Trigger a unique violation, return its ``exc_info`` tuple."""
239+ try:
240+ self.cause_unique_violation()
241+ except IntegrityError as e:
242+ if is_unique_violation(e):
243+ return sys.exc_info()
244+ else:
245+ raise
246
247=== modified file 'src/maasserver/utils/orm.py'
248--- src/maasserver/utils/orm.py 2016-03-28 13:54:47 +0000
249+++ src/maasserver/utils/orm.py 2016-06-17 14:52:22 +0000
250@@ -16,8 +16,10 @@
251 'is_deadlock_failure',
252 'is_retryable_failure',
253 'is_serialization_failure',
254+ 'is_unique_violation',
255 'make_deadlock_failure',
256 'make_serialization_failure',
257+ 'make_unique_violation',
258 'post_commit',
259 'post_commit_do',
260 'psql_array',
261@@ -54,7 +56,11 @@
262 )
263 from django.db.models import Q
264 from django.db.transaction import TransactionManagementError
265-from django.db.utils import OperationalError
266+from django.db.utils import (
267+ DatabaseError,
268+ IntegrityError,
269+ OperationalError,
270+)
271 from django.http import Http404
272 from maasserver.exceptions import (
273 MAASAPIBadRequest,
274@@ -72,6 +78,7 @@
275 from psycopg2.errorcodes import (
276 DEADLOCK_DETECTED,
277 SERIALIZATION_FAILURE,
278+ UNIQUE_VIOLATION,
279 )
280 from twisted.internet.defer import Deferred
281
282@@ -273,6 +280,68 @@
283 return get_psycopg2_deadlock_exception(exception) is not None
284
285
286+def get_psycopg2_unique_violation_exception(exception):
287+ """Return the root-cause if `exception` is a unique violation.
288+
289+ PostgreSQL sets a specific error code, "23505", when a transaction breaks
290+ because of a unique violation.
291+
292+ :return: The underlying `psycopg2.Error` if it's a unique violation, or
293+ `None` if there isn't one.
294+ """
295+ exception = get_psycopg2_exception(exception)
296+ if exception is None:
297+ return None
298+ elif exception.pgcode == UNIQUE_VIOLATION:
299+ return exception
300+ else:
301+ return None
302+
303+
304+def is_unique_violation(exception):
305+ """Does `exception` represent a unique violation?
306+
307+ PostgreSQL sets a specific error code, "23505", when a transaction breaks
308+ because of a unique violation.
309+ """
310+ return get_psycopg2_unique_violation_exception(exception) is not None
311+
312+
313+class UniqueViolation(psycopg2.IntegrityError):
314+ """Explicit serialization failure.
315+
316+ A real unique violation, arising out of psycopg2 (and thus signalled from
317+ the database) would *NOT* be an instance of this class. However, it is not
318+ obvious how to create a `psycopg2.IntegrityError` with ``pgcode`` set to
319+ `UNIQUE_VIOLATION` without subclassing. I suspect only the C interface can
320+ do that.
321+ """
322+ pgcode = UNIQUE_VIOLATION
323+
324+
325+def make_unique_violation():
326+ """Make a serialization exception.
327+
328+ Artificially construct an exception that resembles what Django's ORM would
329+ raise when PostgreSQL fails a transaction because of a unique violation.
330+
331+ :returns: an instance of :py:class:`IntegrityError` that will pass the
332+ `is_unique_violation` predicate.
333+ """
334+ exception = IntegrityError()
335+ exception.__cause__ = UniqueViolation()
336+ assert is_unique_violation(exception)
337+ return exception
338+
339+
340+class RetryTransaction(BaseException):
341+ """An explicit request that the transaction be retried."""
342+
343+
344+class TooManyRetries(Exception):
345+ """A transaction retry has been requested too many times."""
346+
347+
348 def request_transaction_retry():
349 """Raise a serialization exception.
350
351@@ -280,15 +349,24 @@
352 this, and then retrying the transaction, though it may choose to re-raise
353 the error if too many retries have already been attempted.
354
355- :raises OperationalError:
356+ :raise RetryTransaction:
357 """
358- raise make_serialization_failure()
359+ raise RetryTransaction()
360
361
362 def is_retryable_failure(exception):
363- """Does `exception` represent a serialization or deadlock failure?"""
364+ """Does `exception` represent a retryable failure?
365+
366+ This does NOT include requested retries, i.e. `RetryTransaction`.
367+
368+ :param exception: An instance of :class:`DatabaseError` or one of its
369+ subclasses.
370+ """
371 return (
372- is_serialization_failure(exception) or is_deadlock_failure(exception))
373+ is_serialization_failure(exception) or
374+ is_deadlock_failure(exception) or
375+ is_unique_violation(exception)
376+ )
377
378
379 def gen_retry_intervals(base=0.01, rate=2.5, maximum=10.0):
380@@ -341,14 +419,22 @@
381 for _ in range(9):
382 try:
383 return func(*args, **kwargs)
384- except OperationalError as error:
385+ except RetryTransaction:
386+ reset() # Which may do nothing.
387+ sleep(next(intervals))
388+ except DatabaseError as error:
389 if is_retryable_failure(error):
390 reset() # Which may do nothing.
391 sleep(next(intervals))
392 else:
393 raise
394 else:
395- return func(*args, **kwargs)
396+ try:
397+ return func(*args, **kwargs)
398+ except RetryTransaction:
399+ raise TooManyRetries(
400+ "This transaction has already been attempted "
401+ "multiple times; giving up.")
402 return retrier
403
404
405
406=== modified file 'src/maasserver/utils/tests/test_orm.py'
407--- src/maasserver/utils/tests/test_orm.py 2016-05-12 19:07:37 +0000
408+++ src/maasserver/utils/tests/test_orm.py 2016-06-17 14:52:22 +0000
409@@ -26,12 +26,16 @@
410 )
411 from django.db.backends.base.base import BaseDatabaseWrapper
412 from django.db.transaction import TransactionManagementError
413-from django.db.utils import OperationalError
414+from django.db.utils import (
415+ IntegrityError,
416+ OperationalError,
417+)
418 from maasserver.models import Node
419 from maasserver.testing.testcase import (
420 MAASServerTestCase,
421 MAASTransactionServerTestCase,
422 SerializationFailureTestCase,
423+ UniqueViolationTestCase,
424 )
425 from maasserver.utils import orm
426 from maasserver.utils.orm import (
427@@ -46,11 +50,12 @@
428 get_psycopg2_deadlock_exception,
429 get_psycopg2_exception,
430 get_psycopg2_serialization_exception,
431+ get_psycopg2_unique_violation_exception,
432 in_transaction,
433 is_deadlock_failure,
434 is_retryable_failure,
435 is_serialization_failure,
436- make_serialization_failure,
437+ is_unique_violation,
438 post_commit,
439 post_commit_do,
440 post_commit_hooks,
441@@ -81,6 +86,7 @@
442 from psycopg2.errorcodes import (
443 DEADLOCK_DETECTED,
444 SERIALIZATION_FAILURE,
445+ UNIQUE_VIOLATION,
446 )
447 from testtools import ExpectedException
448 from testtools.matchers import (
449@@ -221,6 +227,16 @@
450 SERIALIZATION_FAILURE, error.__cause__.pgcode)
451
452
453+class TestUniqueViolation(UniqueViolationTestCase):
454+ """Detecting UNIQUE_VIOLATION failures."""
455+
456+ def test_unique_violation_detectable_via_error_cause(self):
457+ error = self.assertRaises(
458+ IntegrityError, self.cause_unique_violation)
459+ self.assertEqual(
460+ UNIQUE_VIOLATION, error.__cause__.pgcode)
461+
462+
463 class TestGetPsycopg2Exception(MAASTestCase):
464 """Tests for `get_psycopg2_exception`."""
465
466@@ -281,6 +297,25 @@
467 get_psycopg2_deadlock_exception(exception))
468
469
470+class TestGetPsycopg2UniqueViolationException(MAASTestCase):
471+ """Tests for `get_psycopg2_unique_violation_exception`."""
472+
473+ def test__returns_None_for_plain_psycopg2_error(self):
474+ exception = psycopg2.Error()
475+ self.assertIsNone(get_psycopg2_unique_violation_exception(exception))
476+
477+ def test__returns_None_for_other_error(self):
478+ exception = factory.make_exception()
479+ self.assertIsNone(get_psycopg2_unique_violation_exception(exception))
480+
481+ def test__returns_psycopg2_error_root_cause(self):
482+ exception = Exception()
483+ exception.__cause__ = orm.UniqueViolation()
484+ self.assertIs(
485+ exception.__cause__,
486+ get_psycopg2_unique_violation_exception(exception))
487+
488+
489 class TestIsSerializationFailure(SerializationFailureTestCase):
490 """Tests relating to MAAS's use of SERIALIZABLE isolation."""
491
492@@ -340,6 +375,36 @@
493 self.assertFalse(is_deadlock_failure(error))
494
495
496+class TestIsUniqueViolation(UniqueViolationTestCase):
497+ """Tests relating to MAAS's identification of unique violations."""
498+
499+ def test_detects_integrity_error_with_matching_cause(self):
500+ error = self.assertRaises(
501+ IntegrityError, self.cause_unique_violation)
502+ self.assertTrue(is_unique_violation(error))
503+
504+ def test_rejects_integrity_error_without_matching_cause(self):
505+ error = IntegrityError()
506+ cause = self.patch(error, "__cause__", Exception())
507+ cause.pgcode = factory.make_name("pgcode")
508+ self.assertFalse(is_unique_violation(error))
509+
510+ def test_rejects_integrity_error_with_unrelated_cause(self):
511+ error = IntegrityError()
512+ error.__cause__ = Exception()
513+ self.assertFalse(is_unique_violation(error))
514+
515+ def test_rejects_integrity_error_without_cause(self):
516+ error = IntegrityError()
517+ self.assertFalse(is_unique_violation(error))
518+
519+ def test_rejects_non_integrity_error_with_matching_cause(self):
520+ error = factory.make_exception()
521+ cause = self.patch(error, "__cause__", Exception())
522+ cause.pgcode = UNIQUE_VIOLATION
523+ self.assertFalse(is_unique_violation(error))
524+
525+
526 class TestIsRetryableFailure(MAASTestCase):
527 """Tests relating to MAAS's use of catching retryable failures."""
528
529@@ -351,33 +416,58 @@
530 error = orm.make_deadlock_failure()
531 self.assertTrue(is_retryable_failure(error))
532
533+ def test_detects_unique_violation(self):
534+ error = orm.make_unique_violation()
535+ self.assertTrue(is_retryable_failure(error))
536+
537 def test_rejects_operational_error_without_matching_cause(self):
538 error = OperationalError()
539 cause = self.patch(error, "__cause__", Exception())
540 cause.pgcode = factory.make_name("pgcode")
541 self.assertFalse(is_retryable_failure(error))
542
543+ def test_rejects_integrity_error_without_matching_cause(self):
544+ error = IntegrityError()
545+ cause = self.patch(error, "__cause__", Exception())
546+ cause.pgcode = factory.make_name("pgcode")
547+ self.assertFalse(is_retryable_failure(error))
548+
549 def test_rejects_operational_error_with_unrelated_cause(self):
550 error = OperationalError()
551 error.__cause__ = Exception()
552 self.assertFalse(is_retryable_failure(error))
553
554+ def test_rejects_integrity_error_with_unrelated_cause(self):
555+ error = IntegrityError()
556+ error.__cause__ = Exception()
557+ self.assertFalse(is_retryable_failure(error))
558+
559 def test_rejects_operational_error_without_cause(self):
560 error = OperationalError()
561 self.assertFalse(is_retryable_failure(error))
562
563- def test_rejects_non_operational_error_with_cause_serialization(self):
564+ def test_rejects_integrity_error_without_cause(self):
565+ error = IntegrityError()
566+ self.assertFalse(is_retryable_failure(error))
567+
568+ def test_rejects_non_database_error_with_cause_serialization(self):
569 error = factory.make_exception()
570 cause = self.patch(error, "__cause__", Exception())
571 cause.pgcode = SERIALIZATION_FAILURE
572 self.assertFalse(is_retryable_failure(error))
573
574- def test_rejects_non_operational_error_with_cause_deadlock(self):
575+ def test_rejects_non_database_error_with_cause_deadlock(self):
576 error = factory.make_exception()
577 cause = self.patch(error, "__cause__", Exception())
578 cause.pgcode = DEADLOCK_DETECTED
579 self.assertFalse(is_retryable_failure(error))
580
581+ def test_rejects_non_database_error_with_cause_unique_violation(self):
582+ error = factory.make_exception()
583+ cause = self.patch(error, "__cause__", Exception())
584+ cause.pgcode = UNIQUE_VIOLATION
585+ self.assertFalse(is_retryable_failure(error))
586+
587
588 class TestRetryOnRetryableFailure(SerializationFailureTestCase):
589
590@@ -418,6 +508,36 @@
591 self.assertEqual(sentinel.result, function_wrapped())
592 self.assertThat(function, MockCallsMatch(call(), call()))
593
594+ def test_retries_on_unique_violation(self):
595+ function = self.make_mock_function()
596+ function.side_effect = orm.make_unique_violation()
597+ function_wrapped = retry_on_retryable_failure(function)
598+ self.assertRaises(IntegrityError, function_wrapped)
599+ expected_calls = [call()] * 10
600+ self.assertThat(function, MockCallsMatch(*expected_calls))
601+
602+ def test_retries_on_unique_violation_until_successful(self):
603+ function = self.make_mock_function()
604+ function.side_effect = [orm.make_unique_violation(), sentinel.result]
605+ function_wrapped = retry_on_retryable_failure(function)
606+ self.assertEqual(sentinel.result, function_wrapped())
607+ self.assertThat(function, MockCallsMatch(call(), call()))
608+
609+ def test_retries_on_retry_transaction(self):
610+ function = self.make_mock_function()
611+ function.side_effect = orm.RetryTransaction()
612+ function_wrapped = retry_on_retryable_failure(function)
613+ self.assertRaises(orm.TooManyRetries, function_wrapped)
614+ expected_calls = [call()] * 10
615+ self.assertThat(function, MockCallsMatch(*expected_calls))
616+
617+ def test_retries_on_retry_transaction_until_successful(self):
618+ function = self.make_mock_function()
619+ function.side_effect = [orm.RetryTransaction(), sentinel.result]
620+ function_wrapped = retry_on_retryable_failure(function)
621+ self.assertEqual(sentinel.result, function_wrapped())
622+ self.assertThat(function, MockCallsMatch(call(), call()))
623+
624 def test_passes_args_to_wrapped_function(self):
625 function = lambda a, b: (a, b)
626 function_wrapped = retry_on_retryable_failure(function)
627@@ -450,19 +570,34 @@
628 """Tests for `make_serialization_failure`."""
629
630 def test__makes_a_serialization_failure(self):
631- exception = make_serialization_failure()
632+ exception = orm.make_serialization_failure()
633 self.assertThat(exception, MatchesPredicate(
634 is_serialization_failure, "%r is not a serialization failure."))
635
636
637+class TestMakeDeadlockFailure(MAASTestCase):
638+ """Tests for `make_deadlock_failure`."""
639+
640+ def test__makes_a_deadlock_failure(self):
641+ exception = orm.make_deadlock_failure()
642+ self.assertThat(exception, MatchesPredicate(
643+ is_deadlock_failure, "%r is not a deadlock failure."))
644+
645+
646+class TestMakeUniqueViolation(MAASTestCase):
647+ """Tests for `make_unique_violation`."""
648+
649+ def test__makes_a_unique_violation(self):
650+ exception = orm.make_unique_violation()
651+ self.assertThat(exception, MatchesPredicate(
652+ is_unique_violation, "%r is not a unique violation."))
653+
654+
655 class TestRequestTransactionRetry(MAASTestCase):
656 """Tests for `request_transaction_retry`."""
657
658- def test__raises_a_serialization_failure(self):
659- exception = self.assertRaises(
660- OperationalError, request_transaction_retry)
661- self.assertThat(exception, MatchesPredicate(
662- is_serialization_failure, "%r is not a serialization failure."))
663+ def test__raises_a_retry_transaction_exception(self):
664+ self.assertRaises(orm.RetryTransaction, request_transaction_retry)
665
666
667 class TestGenRetryIntervals(MAASTestCase):