Merge lp:~vila/ols-store-tests/better-reporting into lp:ols-store-tests

[Vincent Ladeuil]

Improve reporting on download errors and increase wait for upload scans.

This improves the reporting when downloading from updown with the url received from cpi (instead of failing later on with a SHAMismatchError which is confusing).

This also increases waiting for upload to succeed to avoid spurious failures with 'Package scan took too long'.

[Fabián Ezequiel Gallina (fgallina)]

LGTM

review: Approve

=== modified file 'olsstore/__init__.py'
--- olsstore/__init__.py	2016-07-11 09:08:32 +0000
+++ olsstore/__init__.py	2017-03-29 10:25:57 +0000
@@ -202,8 +202,17 @@
                 name, download_path))
             return
         logger.info('Downloading {}'.format(name, download_path))
-        # FIXME: Check the status code ! -- vila 2016-05-04
-        download = self.cpi.get(download_url)
+        auth = _macaroon_auth(self.conf)
+        download = self.updown.get(download_url,
+                                   headers={'Authorization': auth})
+        if not download.ok:
+            # This only happens when something is wrong server side and improve
+            # the reporting to help diagnosis. Arguably this should retry
+            # silently but in practice this only happens for wrong reasons.
+            raise StoreError(
+                fmt='Failed to download from {url}: {status} {reason}.',
+                url=download_url, status=download.status_code,
+                reason=download.reason)
         with open(download_path, 'wb') as f:
             # FIXME: Cough, we may want to buffer here (and a progress bar
             # would be nice) -- vila 2016-04-26
@@ -273,7 +282,7 @@
 
 
 class SSOClient(Client):
-    """The Single Sign On server deals with authentification.
+    """The Single Sign On server deals with authentication.
 
     It is used directly or indirectly by other servers.
     """

=== modified file 'olsstore/_upload.py'
--- olsstore/_upload.py	2016-06-27 10:53:30 +0000
+++ olsstore/_upload.py	2017-03-29 10:25:57 +0000
@@ -273,9 +273,9 @@
 
 def get_scan_data(store, status_url):
     """Return metadata about the state of the upload scan process."""
-    # initial retry after 5 seconds
+    # initial retry after SCAN_STATUS_POLL_DELAY seconds
     # linear backoff after that
-    # abort after 5 retries
+    # abort after SCAN_STATUS_POLL_RETRIES retries
     @retry(terminator=is_scan_completed,
            retries=config.SCAN_STATUS_POLL_RETRIES,
            delay=config.SCAN_STATUS_POLL_DELAY,

=== modified file 'olsstore/config.py'
--- olsstore/config.py	2016-06-27 10:53:30 +0000
+++ olsstore/config.py	2017-03-29 10:25:57 +0000
@@ -23,7 +23,7 @@
 
 DEFAULT_SERIES = '16'
 SCAN_STATUS_POLL_DELAY = 5
-SCAN_STATUS_POLL_RETRIES = 5
+SCAN_STATUS_POLL_RETRIES = 10
 UBUNTU_SSO_API_ROOT_URL = 'https://login.ubuntu.com/api/v2/'
 UBUNTU_STORE_API_ROOT_URL = 'https://myapps.developer.ubuntu.com/dev/api/'
 UBUNTU_STORE_SEARCH_ROOT_URL = 'https://search.apps.ubuntu.com/'

=== modified file 'olsstore/tests/tapes/olsstore.tests.test_upload.TestUpload.test_upload_works'
--- olsstore/tests/tapes/olsstore.tests.test_upload.TestUpload.test_upload_works	2017-01-17 17:01:04 +0000
+++ olsstore/tests/tapes/olsstore.tests.test_upload.TestUpload.test_upload_works	2017-03-29 10:25:57 +0000
@@ -12,15 +12,15 @@
         "status_code": 200
     },
     {
-        "content": "{\"status_details_url\": \"https://myapps.developer.staging.ubuntu.com/dev/api/snaps/sIzw8iQ2Ws6tRpbNV1VETWtSPrPIQdJg/builds/9f35b9f6-5e93-4dc9-bfc9-300694b2b3f6/status\", \"status_url\": \"https://myapps.developer.staging.ubuntu.com/dev/api/click-scan-complete/updown/a-snap-id/\", \"success\": true}",
+        "content": "{\"status_details_url\": \"https://dashboard.staging.snapcraft.io/dev/api/snaps/sIzw8iQ2Ws6tRpbNV1VETWtSPrPIQdJg/builds/4f2ffd3a-9597-4719-9d95-746f39e74148/status\", \"status_url\": \"https://myapps.developer.staging.ubuntu.com/dev/api/click-scan-complete/updown/a-snap-id/\", \"success\": true}",
         "status_code": 202
     },
     {
-        "content": "{\"application_url\": \"\", \"completed\": false, \"message\": \"Task state is unknown.\", \"package_name\": null, \"revision\": 42}",
+        "content": "{\"application_url\": \"\", \"completed\": false, \"message\": \"Task 6a081cee-95f6-4aa2-bd9c-7fb18ce6ae51 is waiting for execution.\", \"package_name\": null, \"revision\": 42}",
         "status_code": 200
     },
     {
-        "content": "{\"application_url\": \"https://myapps.developer.staging.ubuntu.com/dev/click-apps/577/\", \"completed\": true, \"message\": \"\", \"package_name\": \"basic\", \"revision\": 42}",
+        "content": "{\"application_url\": \"https://dashboard.staging.snapcraft.io/dev/click-apps/577/\", \"completed\": true, \"message\": \"\", \"package_name\": \"basic\", \"revision\": 42}",
         "status_code": 200
     }
 ]
\ No newline at end of file