Merge lp:~sseman/juju-ci-tools/ssl-cert into lp:juju-ci-tools

[Seman]

Currently, the Python installed on Jenkins servers have different versions (2.7.6 and 2.7.11). The verify_landscape_bundle.py script uses https to access the front page of the Landscape UI. With Python 2.7.9 and above, we explicitly have to tell it to ignore SSL certificate verification or it throws an URLError exception. In order to do this, I used SSLContext and set the SSLContext.verify_mode to ssl.CERT_NONE. But SSLContext is introduced in 2.7.9 and can't be used in version lower than 2.7.9.

I created _get_ssl_ctx() function that returns ssl context based on the Python version that is used. The ssl.create_default_context() is introduced in 2.7.9. If accessing the ssl.create_default_context doesn't generate AttributeError exception we must be using Python 2.7.9 and the function returns SSL context; otherwise, it returns None. Based on the SSL context returned, either I call urllib2.urlopen(url) or urllib2.urlopen(url, context=ssl-context)

I also noticed that the Landscape bundle occasionally fails waiting for the front page to be accessible. I increased the wait_for_http() timeout to 10min.

[lp:~sseman/juju-ci-tools/ssl-cert]

1343. By Seman on 2016-04-06

Increase wait_for_http timeout to 10min.

[Curtis Hovey (sinzui)]

Thank you.

review: Approve (code)

=== modified file 'tests/test_verify_mediawiki_bundle.py'
--- tests/test_verify_mediawiki_bundle.py	2016-04-04 22:32:27 +0000
+++ tests/test_verify_mediawiki_bundle.py	2016-04-06 06:48:02 +0000
@@ -9,8 +9,10 @@
 from assess_min_version import JujuAssertionError
 import tests
 from tests.test_jujupy import FakeJujuClient
+import verify_mediawiki_bundle
 from verify_mediawiki_bundle import(
     assess_mediawiki_bundle,
+    _get_ssl_ctx,
     parse_args,
     verify_services,
     wait_for_http,
@@ -28,30 +30,50 @@
 
     def test_wait_for_http(self):
         fake_res = FakeResponse()
-        with patch('urllib2.urlopen', autospec=True,
-                   return_value=fake_res) as uo_mock:
+        with patch('verify_mediawiki_bundle._get_ssl_ctx', autospec=True,
+                   return_value=None) as ssl_mock:
+            with patch('urllib2.urlopen', autospec=True,
+                       return_value=fake_res) as uo_mock:
                 wait_for_http("example.com")
         uo_mock.assert_called_once_with('example.com')
+        ssl_mock.assert_called_once_with()
+
+    def test_wait_for_http_ssl_ctx(self):
+        fake_res = FakeResponse()
+        with patch('verify_mediawiki_bundle._get_ssl_ctx', autospec=True,
+                   return_value=FakeSslCtx()) as ssl_mock:
+            with patch('urllib2.urlopen', return_value=fake_res) as uo_mock:
+                        wait_for_http("example.com")
+        uo_mock.assert_called_once_with(
+            'example.com', context=ssl_mock.return_value)
+        ssl_mock.assert_called_once_with()
 
     def test_wait_for_http_timeout(self):
         fake_res = FakeResponse()
-        with patch('urllib2.urlopen', autospec=True,
-                   return_value=fake_res):
-            with self.assertRaisesRegexp(
-                    JujuAssertionError, 'example.com is not reachable'):
-                wait_for_http("example.com", timeout=0)
-
-    def test_wait_for_http_httperror(self):
-        with patch('urllib2.urlopen', autospec=True,
-                   return_value=urllib2.HTTPError(
-                       None, None, None, None, None)) as uo_mock:
-            with patch('verify_mediawiki_bundle.until_timeout',
-                       autospec=True, return_value=[1]) as ut_mock:
+        with patch('verify_mediawiki_bundle._get_ssl_ctx', autospec=True,
+                   return_value=None) as ssl_mock:
+            with patch('urllib2.urlopen', autospec=True,
+                       return_value=fake_res):
                 with self.assertRaisesRegexp(
                         JujuAssertionError, 'example.com is not reachable'):
-                    wait_for_http("example.com")
+                    wait_for_http("example.com", timeout=0)
+        ssl_mock.assert_called_once_with()
+
+    def test_wait_for_http_httperror(self):
+        with patch('verify_mediawiki_bundle._get_ssl_ctx', autospec=True,
+                   return_value=None) as ssl_mock:
+            with patch('urllib2.urlopen', autospec=True,
+                       return_value=urllib2.HTTPError(
+                           None, None, None, None, None)) as uo_mock:
+                with patch('verify_mediawiki_bundle.until_timeout',
+                           autospec=True, return_value=[1]) as ut_mock:
+                    with self.assertRaisesRegexp(
+                            JujuAssertionError,
+                            'example.com is not reachable'):
+                        wait_for_http("example.com")
         uo_mock.assert_called_once_with('example.com')
-        self.assertEqual(ut_mock.mock_calls, [call(60)])
+        self.assertEqual(ut_mock.mock_calls, [call(600)])
+        ssl_mock.assert_called_once_with()
 
     def test_verify_services(self):
         client = self.deploy_mediawiki()
@@ -60,8 +82,11 @@
                     'mysql-slave']
         with patch('urllib2.urlopen', autospec=True,
                    return_value=fake_res) as url_mock:
-            verify_services(client, services, text='foo')
+            with patch('verify_mediawiki_bundle._get_ssl_ctx', autospec=True,
+                       return_value=None) as ssl_mock:
+                verify_services(client, services, text='foo')
         url_mock.assert_called_once_with('http://1.example.com')
+        ssl_mock.assert_called_once_with()
 
     def test_verify_service_misconfigured(self):
         client = FakeJujuClient()
@@ -115,6 +140,19 @@
             assess_mediawiki_bundle(client)
         vs_mock.assert_called_once_with(client, services)
 
+    def test_get_ssl_ctx(self):
+        ssl_mock = Mock(spec=['create_default_context'], CERT_NONE=0)
+        ssl_mock.create_default_context.return_value = FakeSslCtx()
+        verify_mediawiki_bundle.ssl = ssl_mock
+        ctx = _get_ssl_ctx()
+        self.assertIs(ctx.check_hostname, False)
+        self.assertEqual(ctx.verify_mode, 0)
+
+    def test_get_ssl_ctx_none(self):
+        verify_mediawiki_bundle.ssl = Mock(spec=[])
+        ctx = _get_ssl_ctx()
+        self.assertIsNone(ctx)
+
     def deploy_mediawiki(self):
         client = FakeJujuClient()
         client.bootstrap()
@@ -126,6 +164,15 @@
         return client
 
 
+class FakeSslCtx:
+    check_hostname = None
+    verify_mode = None
+
+
+class FakeSslOld:
+    pass
+
+
 class FakeResponse:
 
     def __init__(self, code=200):

=== modified file 'verify_landscape_bundle.py'
--- verify_landscape_bundle.py	2016-04-04 22:32:27 +0000
+++ verify_landscape_bundle.py	2016-04-06 06:48:02 +0000
@@ -4,6 +4,8 @@
 import logging
 import sys
 
+
+from utility import configure_logging
 from verify_mediawiki_bundle import (
     parse_args,
     verify_services,
@@ -23,6 +25,7 @@
 
 def main(argv=None):
     args = parse_args(argv)
+    configure_logging(args.verbose)
     assess_landscape_bundle(args.client)
 
 

=== modified file 'verify_mediawiki_bundle.py'
--- verify_mediawiki_bundle.py	2016-04-05 01:08:36 +0000
+++ verify_mediawiki_bundle.py	2016-04-06 06:48:02 +0000
@@ -5,6 +5,7 @@
 import logging
 import pickle
 import sys
+import ssl
 from time import sleep
 import urllib2
 
@@ -21,10 +22,24 @@
 log = logging.getLogger("mediawiki_bundle")
 
 
-def wait_for_http(url, timeout=60):
+def _get_ssl_ctx():
+    try:
+        ctx = ssl.create_default_context()
+    except AttributeError:
+        return None
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
+def wait_for_http(url, timeout=600):
+    ctx = _get_ssl_ctx()
     for _ in until_timeout(timeout):
         try:
-            req = urllib2.urlopen(url)
+            if ctx is None:
+                req = urllib2.urlopen(url)
+            else:
+                req = urllib2.urlopen(url, context=ctx)
             if 200 == req.getcode():
                 break
         except (urllib2.URLError, urllib2.HTTPError):