MAAS

  1. Bug #1908452
  2. Comment #17

Comment 17 for bug 1908452

Revision history for this message
dann frazier (dannf) wrote :

The issue has reproduced again - here's a backtrace I was able to capture from the hung regiond process:

Thread 0x00007fa414f8d700 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 306 in wait
  File "/usr/lib/python3.8/threading.py", line 558 in wait
  File "/usr/lib/python3/dist-packages/crochet/_eventloop.py", line 189 in _result
  File "/usr/lib/python3/dist-packages/crochet/_eventloop.py", line 229 in wait
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 126 in wrapper
  File "/usr/lib/python3/dist-packages/maasserver/bootresources.py", line 1359 in _import_resources_without_lock
  File "/usr/lib/python3/dist-packages/maasserver/bootresources.py", line 1429 in _import_resources_with_lock
  File "/usr/lib/python3/dist-packages/maasserver/utils/__init__.py", line 193 in call_with_lock
  File "/usr/lib/python3/dist-packages/maasserver/utils/orm.py", line 692 in call_with_connection
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 192 in wrapper
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 860 in callInContext
  File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 85 in callWithContext
  File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 122 in callWithContext
  File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 266 in <lambda>
  File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 250 in inContext
  File "/usr/lib/python3/dist-packages/twisted/_threads/_team.py", line 190 in doWork
  File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 46 in work
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007fa415bdd700 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 302 in wait
  File "/usr/lib/python3.8/queue.py", line 170 in get
  File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007fa41641e700 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 302 in wait
  File "/usr/lib/python3.8/queue.py", line 170 in get
  File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007fa416c5f700 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 302 in wait
  File "/usr/lib/python3.8/queue.py", line 170 in get
  File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007fa417460700 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 302 in wait
  File "/usr/lib/python3.8/queue.py", line 170 in get
  File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007fa417db5700 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 302 in wait
  File "/usr/lib/python3.8/queue.py", line 170 in get
  File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007fa4185b6700 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 306 in wait
  File "/usr/lib/python3.8/threading.py", line 558 in wait
  File "/usr/lib/python3/dist-packages/crochet/_eventloop.py", line 189 in _result
  File "/usr/lib/python3/dist-packages/crochet/_eventloop.py", line 229 in wait
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 126 in wrapper
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 192 in wrapper
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 860 in callInContext
  File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 85 in callWithContext
  File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 122 in callWithContext
  File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 266 in <lambda>
  File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 250 in inContext
  File "/usr/lib/python3/dist-packages/twisted/_threads/_team.py", line 190 in doWork
  File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 46 in work
  File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Current thread 0x00007fa420670740 (most recent call first):
  File "/usr/lib/python3/dist-packages/OpenSSL/SSL.py", line 1914 in do_handshake
  File "/usr/lib/python3/dist-packages/urllib3/contrib/pyopenssl.py", line 485 in wrap_socket
  File "/usr/lib/python3/dist-packages/urllib3/util/ssl_.py", line 370 in ssl_wrap_socket
  File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 366 in connect
  File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 996 in _validate_conn
  File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 376 in _make_request
  File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 665 in urlopen
  File "/usr/lib/python3/dist-packages/requests/adapters.py", line 439 in send
  File "/usr/lib/python3/dist-packages/requests/sessions.py", line 646 in send
  File "/usr/lib/python3/dist-packages/requests/sessions.py", line 533 in request
  File "/usr/lib/python3/dist-packages/requests_unixsocket/__init__.py", line 46 in request
  File "/usr/lib/python3/dist-packages/requests_unixsocket/__init__.py", line 51 in get
  File "/usr/lib/python3/dist-packages/simplestreams/contentsource.py", line 381 in __init__
  File "/usr/lib/python3/dist-packages/simplestreams/mirrors/__init__.py", line 186 in url_reader_factory
  File "/usr/lib/python3/dist-packages/simplestreams/contentsource.py", line 119 in _open
  File "/usr/lib/python3/dist-packages/simplestreams/contentsource.py", line 131 in open
  File "/usr/lib/python3/dist-packages/simplestreams/contentsource.py", line 74 in __enter__
  File "/usr/lib/python3/dist-packages/simplestreams/mirrors/__init__.py", line 41 in read_json
  File "/usr/lib/python3/dist-packages/simplestreams/mirrors/__init__.py", line 85 in sync
  File "/usr/lib/python3/dist-packages/provisioningserver/import_images/download_descriptions.py", line 201 in sync
  File "/usr/lib/python3/dist-packages/provisioningserver/import_images/download_descriptions.py", line 333 in download_image_descriptions
  File "/usr/lib/python3/dist-packages/provisioningserver/import_images/download_descriptions.py", line 343 in download_all_image_descriptions
  File "/usr/lib/python3/dist-packages/maasserver/bootsources.py", line 360 in cache_boot_sources
  File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1418 in _inlineCallbacks
  File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1475 in gotResult
  File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654 in _runCallbacks
  File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 568 in _startRunCallbacks
  File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 460 in callback
  File "/usr/lib/python3/dist-packages/twisted/internet/asyncioreactor.py", line 290 in run
  File "/usr/lib/python3/dist-packages/twisted/internet/asyncioreactor.py", line 267 in run
  File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 312 in runReactorWithLogging
  File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 398 in startReactor
  File "/usr/lib/python3/dist-packages/twisted/scripts/_twistd_unix.py", line 268 in postApplication
  File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 385 in run
  File "/usr/lib/python3/dist-packages/provisioningserver/server.py", line 90 in runService
  File "/usr/lib/python3/dist-packages/maasserver/server.py", line 35 in runWorkerServices
  File "/usr/lib/python3/dist-packages/maasserver/server.py", line 78 in run
  File "/usr/sbin/regiond", line 11 in <module>