Merge ~ubuntu-release/britney/+git/britney2-ubuntu:wip/httplib2 into ~ubuntu-release/britney/+git/britney2-ubuntu:master

[Iain Lane]

[Iain Lane (laney)]

Nah, I'll make this use requests, stand by!

review: Disapprove

Unmerged commits

acb2bda... by Iain Lane on 2020-03-24

autopkgtest: Re-use the HTTP connection between requests to swift

diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py
index 669d4a9..77ae90f 100644
--- a/britney2/policies/autopkgtest.py
+++ b/britney2/policies/autopkgtest.py
@@ -24,7 +24,7 @@ import io
 import re
 import sys
 import urllib.parse
-from urllib.request import urlopen
+import httplib2
 
 import apt_pkg
 import amqplib.client_0_8 as amqp
@@ -84,6 +84,8 @@ class AutopkgtestPolicy(BasePolicy):
         else:
             self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache')
 
+        self.http = httplib2.Http(timeout=30)
+
         try:
             self.options.adt_ppas = self.options.adt_ppas.strip().split()
         except AttributeError:
@@ -484,29 +486,22 @@ class AutopkgtestPolicy(BasePolicy):
         # request new results from swift
         url = os.path.join(swift_url, self.swift_container)
         url += '?' + urllib.parse.urlencode(query)
-        try:
-            f = urlopen(url, timeout=30)
-            if f.getcode() == 200:
-                result_paths = f.read().decode().strip().splitlines()
-            elif f.getcode() == 204:  # No content
-                result_paths = []
-            else:
-                # we should not ever end up here as we expect a HTTPError in
-                # other cases; e. g. 3XX is something that tells us to adjust
-                # our URLS, so fail hard on those
-                raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' %
-                                          (url, f.getcode()))
-            f.close()
-        except IOError as e:
+
+        (resp, content) = self.http.request(url, 'GET')
+        if resp.status == 200:
+            result_paths = content.decode().strip().splitlines()
+        elif resp.status == 204:  # No content
+            result_paths = []
+        elif resp.status == 401:
             # 401 "Unauthorized" is swift's way of saying "container does not exist"
-            if hasattr(e, 'code') and e.code == 401:
-                self.log('fetch_swift_results: %s does not exist yet or is inaccessible' % url)
-                return
+            self.log('fetch_swift_results: %s does not exist yet or is inaccessible' % url)
+            return
+        else:
             # Other status codes are usually a transient
             # network/infrastructure failure. Ignoring this can lead to
             # re-requesting tests which we already have results for, so
             # fail hard on this and let the next run retry.
-            self.log('FATAL: Failure to fetch swift results from %s: %s' % (url, str(e)), 'E')
+            self.log('FATAL: Failure to fetch swift results from %s: got error code %s' % (url, f.status))
             sys.exit(1)
 
         for p in result_paths:
@@ -520,20 +515,15 @@ class AutopkgtestPolicy(BasePolicy):
 
         Remove matching pending_tests entries.
         '''
-        try:
-            f = urlopen(url, timeout=30)
-            if f.getcode() == 200:
-                tar_bytes = io.BytesIO(f.read())
-                f.close()
-            else:
-                raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' %
-                                          (url, f.getcode()))
-        except IOError as e:
-            self.log('Failure to fetch %s: %s' % (url, str(e)), 'E')
-            # we tolerate "not found" (something went wrong on uploading the
-            # result), but other things indicate infrastructure problems
-            if hasattr(e, 'code') and e.code == 404:
-                return
+        (resp, content) = self.http.request(url, 'GET')
+        if resp.status == 200:
+            tar_bytes = io.BytesIO(content)
+        # we tolerate "not found" (something went wrong on uploading the
+        # result), but other things indicate infrastructure problems
+        elif resp.status == 404:
+            return
+        else:
+            self.log('Failure to fetch %s: error code %s' % (url, resp.status))
             sys.exit(1)
 
         try: