Comment 5 for bug 2033505

I tried to update to 3.4.1/stable and the error persists, always 2 out of 3 regions have an error, at times even all 3 have an error, we had to rollback to 3.4.0

follow the error below

2024-03-26 03:17:31 maasserver.ipc: [critical] Failed to update regiond's processes and endpoints; prod-web-maas-region3:pid=944396 record's may be out of date
 Traceback (most recent call last):
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/asyncioreactor.py", line 271, in _onTimer
     self.runUntilCurrent()
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/base.py", line 991, in runUntilCurrent
     call.func(*call.args, **call.kw)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 700, in errback
     self._startRunCallbacks(fail)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 763, in _startRunCallbacks
     self._runCallbacks()
 --- <exception caught here> ---
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 857, in _runCallbacks
     current.result = callback( # type: ignore[misc]
   File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/ipc.py", line 622, in ignore_cancel
     failure.trap(CancelledError)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/failure.py", line 451, in trap
     self.raiseException()
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/failure.py", line 475, in raiseException
     raise self.value.with_traceback(self.tb)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 244, in inContext
     result = inContext.theWork() # type: ignore[attr-defined]
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 260, in <lambda>
     inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/context.py", line 117, in callWithContext
     return self.currentContext().callWithContext(ctx, func, *args, **kw)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/context.py", line 82, in callWithContext
     return func(*args, **kw)
   File "/snap/maas/34087/lib/python3.10/site-packages/provisioningserver/utils/twisted.py", line 856, in callInContext
     return func(*args, **kwargs)
   File "/snap/maas/34087/lib/python3.10/site-packages/provisioningserver/utils/twisted.py", line 203, in wrapper
     result = func(*args, **kwargs)
   File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/utils/orm.py", line 771, in call_within_transaction
     return func_outside_txn(*args, **kwargs)
   File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/utils/orm.py", line 587, in retrier
     return func(*args, **kwargs)
   File "/usr/lib/python3.10/contextlib.py", line 79, in inner
     return func(*args, **kwds)
   File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/ipc.py", line 590, in _update
     self._updateConnections(process, conn["rpc"]["connections"].copy())
   File "/snap/maas/34087/lib/python3.10/site-packages/provisioningserver/utils/twisted.py", line 203, in wrapper
     result = func(*args, **kwargs)
   File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/ipc.py", line 518, in _updateConnections
     db_conn, _ = self._registerConnection(
   File "/snap/maas/34087/lib/python3.10/site-packages/provisioningserver/utils/twisted.py", line 203, in wrapper
     result = func(*args, **kwargs)
   File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/ipc.py", line 391, in _registerConnection
     connection, created = RegionRackRPCConnection.objects.get_or_create(
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/manager.py", line 85, in manager_method
     return getattr(self.get_queryset(), name)(*args, **kwargs)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/query.py", line 588, in get_or_create
     return self.create(**params), True
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/query.py", line 453, in create
     obj.save(force_insert=True, using=self.db)
   File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/models/cleansave.py", line 48, in save
     return super().save(*args, **kwargs)
   File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/models/timestampedmodel.py", line 78, in save
     return super().save(*args, **kwargs)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/base.py", line 739, in save
     self.save_base(using=using, force_insert=force_insert,
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/base.py", line 776, in save_base
     updated = self._save_table(
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/base.py", line 881, in _save_table
     results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/base.py", line 919, in _do_insert
     return manager._insert(
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/manager.py", line 85, in manager_method
     return getattr(self.get_queryset(), name)(*args, **kwargs)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/query.py", line 1270, in _insert
     return query.get_compiler(using=using).execute_sql(returning_fields)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/sql/compiler.py", line 1416, in execute_sql
     cursor.execute(sql, params)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 66, in execute
     return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers
     return executor(sql, params, many, context)
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 79, in _execute
     with self.db.wrap_database_errors:
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/utils.py", line 90, in __exit__
     raise dj_exc_value.with_traceback(traceback) from exc_value
   File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 84, in _execute
     return self.cursor.execute(sql, params)
 django.db.utils.IntegrityError: duplicate key value violates unique constraint "maasserver_regionrackrpcconne_endpoint_id_52afd589b0be380e_uniq"
 DETAIL: Key (endpoint_id, rack_controller_id)=(51289, 5934) already exists.