Merge lp:~cjwatson/launchpad/twisted-16.5.0 into lp:launchpad

[Colin Watson]

Upgrade to Twisted 16.5.0.

As well as the various upgrades associated with Twisted itself, we also need lazr.sshserver 0.1.5 and txfixtures 0.4.1 for compatibility.

Going any further than 16.5.0 will require some performance testing due to https://twistedmatrix.com/trac/ticket/8079; we'll probably need to either monkey-patch gmpy into place ourselves or implement the suggested DH_compute_key changes in Twisted.

Since cryptography is rather stricter than PyCrypto, we should scan for registered SSH public keys it doesn't like before landing this and notify their owners: for example, very short DSA keys will now be refused.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'constraints.txt'
--- constraints.txt	2018-02-21 15:25:48 +0000
+++ constraints.txt	2018-02-26 12:48:14 +0000
@@ -228,6 +228,9 @@
 amqp==1.4.9
 amqplib==1.0.2
 anyjson==0.3.3
+appdirs==1.4.3
+asn1crypto==0.23.0
+attrs==17.3.0
 auditor==0.0.3
 auditorclient==0.0.4
 auditorfixture==0.0.7
@@ -238,7 +241,10 @@
 bson==0.3.3
 bzr==2.6.0.lp.2
 celery==3.1.18
+cffi==1.11.2
 Chameleon==2.11
+constantly==15.1.0
+cryptography==2.1.4
 cssselect==0.9.1
 cssutils==0.9.10
 d2to1==0.2.12
@@ -248,18 +254,23 @@
 dnspython==1.10.0
 dulwich==0.18.6
 elementtree==1.2.6-20050316
+enum34==1.1.6
 epydoc==3.0.1
 extras==1.0.0
 feedparser==4.1
 feedvalidator==0.0.0DEV-r1049
 fixtures==3.0.0
 FormEncode==1.2.4
+gmpy==1.17
 grokcore.component==1.6
 html5browser==0.0.9
 httmock==1.2.3
 httplib2==0.8
+idna==2.6
 imagesize==0.7.1
 importlib==1.0.2
+incremental==17.5.0
+ipaddress==1.0.18
 ipython==0.13.2
 iso8601==0.1.12
 jsautobuild==0.2
@@ -277,7 +288,7 @@
 lazr.restful==0.20.1
 lazr.restfulclient==0.13.2
 lazr.smtptest==1.3
-lazr.sshserver==0.1.3
+lazr.sshserver==0.1.7
 lazr.testing==0.1.1
 lazr.uri==1.0.3
 libnacl==1.3.6
@@ -302,13 +313,16 @@
 pbr==0.11.1
 pgbouncer==0.0.8
 prettytable==0.7.2
+psutil==5.4.2
 psycopg2==2.6.1
-pyasn1==0.1.6
+pyasn1==0.4.2
+pyasn1-modules==0.2.1
+pycparser==2.18
 pycrypto==2.6
 pygpgme==0.2
 pyinotify==0.9.4
 pymacaroons==0.9.2
-pyOpenSSL==0.13
+pyOpenSSL==17.5.0
 pystache==0.5.3
 python-dateutil==1.5
 python-debian==0.1.23
@@ -324,11 +338,12 @@
 rabbitfixture==0.3.6
 requests==2.7.0
 requests-toolbelt==0.6.2
+service-identity==17.0.0
 setproctitle==1.1.7
 setuptools-git==1.2
 simplejson==3.8.2
 SimpleTAL==4.3
-six==1.9.0
+six==1.11.0
 snowballstemmer==1.2.1
 soupmatchers==0.2
 sphinxcontrib-websupport==1.0.1
@@ -339,14 +354,9 @@
 testresources==0.2.7
 testscenarios==0.4
 timeline==0.0.3
-# Build of lp:~canonical-launchpad-branches/twisted:lp-backport.
-# p1    Support diffie-hellman-group14-sha1 key exchange in conch.ssh.
-# p2    Add diffie-hellman-group-exchange-sha256 to twisted.conch.ssh.
-#       Add support in twisted.conch.ssh for hmac-sha2-256 and hmac-sha2-512.
-# post3 Use PEP 440-compliant version.
-Twisted==13.0.0post3
+Twisted[conch,tls]==16.5.0
 txAMQP==0.6.2
-txfixtures==0.1.4
+txfixtures==0.4.1
 txlongpoll==0.2.12
 txlongpollfixture==0.1.3
 txpkgupload==0.2
 
=== modified file 'lib/lp/codehosting/tests/test_acceptance.py'
--- lib/lp/codehosting/tests/test_acceptance.py	2016-12-31 05:01:10 +0000
+++ lib/lp/codehosting/tests/test_acceptance.py	2018-02-26 12:48:14 +0000
@@ -5,7 +5,6 @@
 
 __metaclass__ = type
 
-import atexit
 import os
 import re
 import signal
@@ -156,7 +155,6 @@
         tac_handler = SSHServerLayer.getTacHandler()
         tac_handler.setUp()
         SSHServerLayer._reset()
-        atexit.register(tac_handler.tearDown)
         forker = SSHServerLayer.getForker()
         forker.setUp()
 
 
=== modified file 'lib/lp/services/librarianserver/testing/server.py'
--- lib/lp/services/librarianserver/testing/server.py	2016-06-02 17:52:38 +0000
+++ lib/lp/services/librarianserver/testing/server.py	2018-02-26 12:48:14 +0000
@@ -155,20 +155,24 @@
             return
         chunks = self.getLogChunks()
         # A typical startup: upload, download, restricted up, restricted down.
-        #2010-10-20 14:28:21+0530 [-] Log opened.
-        #2010-10-20 14:28:21+0530 [-] twistd 10.1.0 (/usr/bin/python 2.6.5) starting up.
-        #2010-10-20 14:28:21+0530 [-] reactor class: twisted.internet.selectreactor.SelectReactor.
-        #2010-10-20 14:28:21+0530 [-] lp.services.librarianserver.libraryprotocol.FileUploadFactory starting on 59090
-        #2010-10-20 14:28:21+0530 [-] Starting factory <lp.services.librarianserver.libraryprotocol.FileUploadFactory instance at 0x6f8ff38>
-        #2010-10-20 14:28:21+0530 [-] twisted.web.server.Site starting on 58000
-        #2010-10-20 14:28:21+0530 [-] Starting factory <twisted.web.server.Site instance at 0x6fb2638>
-        #2010-10-20 14:28:21+0530 [-] lp.services.librarianserver.libraryprotocol.FileUploadFactory starting on 59095
-        #2010-10-20 14:28:21+0530 [-] Starting factory <lp.services.librarianserver.libraryprotocol.FileUploadFactory instance at 0x6fb25f0>
-        #2010-10-20 14:28:21+0530 [-] twisted.web.server.Site starting on 58005
-        self.upload_port = int(chunks[3].split()[-1])
-        self.download_port = int(chunks[5].split()[-1])
-        self.restricted_upload_port = int(chunks[7].split()[-1])
-        self.restricted_download_port = int(chunks[9].split()[-1])
+        #2017-12-09 06:33:13+0530 [-] Loading /home/ubuntu/launchpad/lp-branches/devel/daemons/librarian.tac...
+        #2017-12-09 06:33:19+0530 [-] Loaded.
+        #2017-12-09 06:33:19+0530 [-] twistd 16.5.0 (/home/ubuntu/launchpad/lp-branches/devel/env/bin/python2.7 2.7.12) starting up.
+        #2017-12-09 06:33:19+0530 [-] reactor class: twisted.internet.epollreactor.EPollReactor.
+        #2017-12-09 06:33:19+0530 [-] FileUploadFactory starting on 39851
+        #2017-12-09 06:33:19+0530 [-] Starting factory <lp.services.librarianserver.libraryprotocol.FileUploadFactory instance at 0xf0ef80ac>
+        #2017-12-09 06:33:19+0530 [-] Site starting on 45355
+        #2017-12-09 06:33:19+0530 [-] Starting factory <twisted.web.server.Site instance at 0xf0ef848c>
+        #2017-12-09 06:33:19+0530 [-] FileUploadFactory starting on 42150
+        #2017-12-09 06:33:19+0530 [-] Starting factory <lp.services.librarianserver.libraryprotocol.FileUploadFactory instance at 0xf0ef856c>
+        #2017-12-09 06:33:19+0530 [-] Site starting on 37341
+        #2017-12-09 06:33:19+0530 [-] Starting factory <twisted.web.server.Site instance at 0xf0ef85ec>
+        #2017-12-09 06:33:19+0530 [-] Not using upstream librarian
+        #2017-12-09 06:33:19+0530 [-] daemon ready!
+        self.upload_port = int(chunks[4].split()[-1])
+        self.download_port = int(chunks[6].split()[-1])
+        self.restricted_upload_port = int(chunks[8].split()[-1])
+        self.restricted_download_port = int(chunks[10].split()[-1])
         self.service_config = dedent("""\
             [librarian_server]
             root: %s
 
=== modified file 'lib/lp/services/twistedsupport/tests/test_gracefulshutdown.py'
--- lib/lp/services/twistedsupport/tests/test_gracefulshutdown.py	2015-10-14 15:22:01 +0000
+++ lib/lp/services/twistedsupport/tests/test_gracefulshutdown.py	2018-02-26 12:48:14 +0000
@@ -99,7 +99,7 @@
 
     def make_dummy_http_request(self):
         """Make a dummy HTTP request for tests."""
-        return http.Request('fake channel', True)
+        return http.Request(http.HTTPChannel(), True)
 
     def test_200_when_available(self):
         """When the factory is available a 200 response is generated."""
 
=== modified file 'setup.py'
--- setup.py	2017-12-22 09:59:33 +0000
+++ setup.py	2018-02-26 12:48:14 +0000
@@ -219,7 +219,7 @@
         'testtools',
         'timeline',
         'transaction',
-        'Twisted',
+        'Twisted[conch,tls]',
         'txfixtures',
         'txlongpoll',
         'txlongpollfixture',