Merge lp:~cjwatson/launchpad/snap-channels-store-client into lp:launchpad

[Colin Watson]

Add model and store client support for automatically releasing snap packages.

Add model and store client support for automatically releasing snap packages.

[Thomi Richards (thomir-deactivatedaccount)]

LGTM - one comment below for your consideration.

[Thomi Richards (thomir-deactivatedaccount)]

Thanks for the fix - looks great.

review: Approve

=== modified file 'configs/development/launchpad-lazr.conf'
--- configs/development/launchpad-lazr.conf	2016-05-18 00:33:18 +0000
+++ configs/development/launchpad-lazr.conf	2016-07-12 15:02:33 +0000
@@ -190,6 +190,7 @@
 builder_proxy_auth_api_endpoint: http://snap-proxy.launchpad.dev:8080/tokens
 builder_proxy_host: snap-proxy.launchpad.dev
 builder_proxy_port: 3128
+store_search_url: https://search.apps.ubuntu.com/
 tools_source: deb http://ppa.launchpad.net/snappy-dev/snapcraft-daily/ubuntu %(series)s main
 
 [txlongpoll]

=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf	2016-06-20 20:18:11 +0000
+++ lib/lp/services/config/schema-lazr.conf	2016-07-12 15:02:33 +0000
@@ -1804,6 +1804,9 @@
 # The store's upload URL endpoint.
 store_upload_url: none
 
+# The store's search URL endpoint.
+store_search_url: none
+
 [process-job-source-groups]
 # This section is used by cronscripts/process-job-source-groups.py.
 dbuser: process-job-source-groups

=== modified file 'lib/lp/snappy/interfaces/snap.py'
--- lib/lp/snappy/interfaces/snap.py	2016-07-05 10:20:51 +0000
+++ lib/lp/snappy/interfaces/snap.py	2016-07-12 15:02:33 +0000
@@ -421,6 +421,13 @@
             "Serialized secrets issued by the store and the login service to "
             "authorize uploads of this snap package."))
 
+    store_channels = List(
+        value_type=TextLine(), title=_("Store channels"),
+        required=False, readonly=False,
+        description=_(
+            "Channels to release this snap package to after uploading it to "
+            "the store."))
+
 
 class ISnapAdminAttributes(Interface):
     """`ISnap` attributes that can be edited by admins.

=== modified file 'lib/lp/snappy/interfaces/snapstoreclient.py'
--- lib/lp/snappy/interfaces/snapstoreclient.py	2016-06-21 14:51:06 +0000
+++ lib/lp/snappy/interfaces/snapstoreclient.py	2016-07-12 15:02:33 +0000
@@ -8,11 +8,14 @@
 __metaclass__ = type
 __all__ = [
     'BadRefreshResponse',
+    'BadReleaseResponse',
     'BadRequestPackageUploadResponse',
     'BadScanStatusResponse',
+    'BadSearchResponse',
     'BadUploadResponse',
     'ISnapStoreClient',
     'NeedsRefreshResponse',
+    'ReleaseFailedResponse',
     'ScanFailedResponse',
     'UnauthorizedUploadResponse',
     'UploadNotScannedYetResponse',
@@ -53,6 +56,18 @@
     pass
 
 
+class BadSearchResponse(Exception):
+    pass
+
+
+class BadReleaseResponse(Exception):
+    pass
+
+
+class ReleaseFailedResponse(Exception):
+    pass
+
+
 class ISnapStoreClient(Interface):
     """Interface for the API provided by the snap store."""
 
@@ -91,6 +106,26 @@
             scanned the upload.
         :raises BadScanStatusResponse: if the store failed to scan the
             upload.
-        :return: A URL on the store with further information about this
-            upload.
+        :return: A tuple of (`url`, `revision`), where `url` is a URL on the
+            store with further information about this upload, and `revision`
+            is the store revision number for the upload or None.
+        """
+
+    def listChannels():
+        """Fetch the current list of channels from the store.
+
+        :raises BadSearchResponse: if the attempt to fetch the list of
+            channels from the store fails.
+        :return: A list of dictionaries, one per channel, each of which
+            contains at least "name" and "display_name" keys.
+        """
+
+    def release(snapbuild, revision):
+        """Tell the store to release a snap build to specified channels.
+
+        :param snapbuild: The `ISnapBuild` to release.
+        :param revision: The revision returned by the store when uploading
+            the build.
+        :raises BadReleaseResponse: if the store failed to release the
+            build.
         """

=== modified file 'lib/lp/snappy/model/snap.py'
--- lib/lp/snappy/model/snap.py	2016-07-05 10:20:51 +0000
+++ lib/lp/snappy/model/snap.py	2016-07-12 15:02:33 +0000
@@ -185,12 +185,14 @@
 
     store_secrets = JSON('store_secrets', allow_none=True)
 
+    store_channels = JSON('store_channels', allow_none=True)
+
     def __init__(self, registrant, owner, distro_series, name,
                  description=None, branch=None, git_ref=None, auto_build=False,
                  auto_build_archive=None, auto_build_pocket=None,
                  require_virtualized=True, date_created=DEFAULT,
                  private=False, store_upload=False, store_series=None,
-                 store_name=None, store_secrets=None):
+                 store_name=None, store_secrets=None, store_channels=None):
         """Construct a `Snap`."""
         super(Snap, self).__init__()
         self.registrant = registrant
@@ -211,6 +213,7 @@
         self.store_series = store_series
         self.store_name = store_name
         self.store_secrets = store_secrets
+        self.store_channels = store_channels
 
     @property
     def valid_webhook_event_types(self):
@@ -506,7 +509,7 @@
             auto_build_archive=None, auto_build_pocket=None,
             require_virtualized=True, processors=None, date_created=DEFAULT,
             private=False, store_upload=False, store_series=None,
-            store_name=None, store_secrets=None):
+            store_name=None, store_secrets=None, store_channels=None):
         """See `ISnapSet`."""
         if not registrant.inTeam(owner):
             if owner.is_team:
@@ -535,7 +538,7 @@
             require_virtualized=require_virtualized, date_created=date_created,
             private=private, store_upload=store_upload,
             store_series=store_series, store_name=store_name,
-            store_secrets=store_secrets)
+            store_secrets=store_secrets, store_channels=store_channels)
         store.add(snap)
 
         if processors is None:

=== modified file 'lib/lp/snappy/model/snapstoreclient.py'
--- lib/lp/snappy/model/snapstoreclient.py	2016-06-27 13:19:15 +0000
+++ lib/lp/snappy/model/snapstoreclient.py	2016-07-12 15:02:33 +0000
@@ -10,25 +10,40 @@
     'SnapStoreClient',
     ]
 
+import json
+try:
+    from json.decoder import JSONDecodeError
+except ImportError:
+    JSONDecodeError = ValueError
 import string
+import time
+from urlparse import urlsplit
 
 from lazr.restful.utils import get_current_browser_request
 from pymacaroons import Macaroon
 import requests
 from requests_toolbelt import MultipartEncoder
+from zope.component import getUtility
 from zope.interface import implementer
+from zope.security.proxy import removeSecurityProxy
 
 from lp.services.config import config
+from lp.services.features import getFeatureFlag
+from lp.services.memcache.interfaces import IMemcacheClient
+from lp.services.scripts import log
 from lp.services.timeline.requesttimeline import get_request_timeline
 from lp.services.timeout import urlfetch
 from lp.services.webapp.url import urlappend
 from lp.snappy.interfaces.snapstoreclient import (
     BadRefreshResponse,
+    BadReleaseResponse,
     BadRequestPackageUploadResponse,
     BadScanStatusResponse,
+    BadSearchResponse,
     BadUploadResponse,
     ISnapStoreClient,
     NeedsRefreshResponse,
+    ReleaseFailedResponse,
     ScanFailedResponse,
     UnauthorizedUploadResponse,
     UploadNotScannedYetResponse,
@@ -99,10 +114,28 @@
         return r
 
 
+# Hardcoded fallback.
+_default_store_channels = [
+    {"name": "candidate", "display_name": "Candidate"},
+    {"name": "edge", "display_name": "Edge"},
+    {"name": "beta", "display_name": "Beta"},
+    {"name": "stable", "display_name": "Stable"},
+    ]
+
+
 @implementer(ISnapStoreClient)
 class SnapStoreClient:
     """A client for the API provided by the snap store."""
 
+    @staticmethod
+    def _getTimeline():
+        # XXX cjwatson 2016-06-29: This can be simplified once jobs have
+        # timeline support.
+        request = get_current_browser_request()
+        if request is None:
+            return None
+        return get_request_timeline(request)
+
     def requestPackageUploadPermission(self, snappy_series, snap_name):
         assert config.snappy.store_url is not None
         request_url = urlappend(config.snappy.store_url, "dev/api/acl/")
@@ -203,6 +236,7 @@
 
     @classmethod
     def refreshDischargeMacaroon(cls, snap):
+        """See `ISnapStoreClient`."""
         assert config.launchpad.openid_provider_root is not None
         assert snap.store_secrets is not None
         refresh_url = urlappend(
@@ -222,6 +256,7 @@
 
     @classmethod
     def checkStatus(cls, status_url):
+        """See `ISnapStoreClient`."""
         try:
             response = urlfetch(status_url)
             response_data = response.json()
@@ -232,7 +267,9 @@
                 elif not response_data["application_url"]:
                     raise ScanFailedResponse(response_data["message"])
                 else:
-                    return response_data["application_url"]
+                    return (
+                        response_data["application_url"],
+                        response_data["revision"])
             else:
                 # New status format.
                 if not response_data["processed"]:
@@ -241,7 +278,88 @@
                     error_message = "\n".join(
                         error["message"] for error in response_data["errors"])
                     raise ScanFailedResponse(error_message)
+                elif not response_data["can_release"]:
+                    return response_data["url"], None
                 else:
-                    return response_data["url"]
+                    return response_data["url"], response_data["revision"]
         except requests.HTTPError as e:
             raise BadScanStatusResponse(e.args[0])
+
+    @classmethod
+    def listChannels(cls):
+        """See `ISnapStoreClient`."""
+        if config.snappy.store_search_url is None:
+            return _default_store_channels
+        channels = None
+        memcache_client = getUtility(IMemcacheClient)
+        search_host = urlsplit(config.snappy.store_search_url).hostname
+        memcache_key = ("%s:channels" % search_host).encode("UTF-8")
+        cached_channels = memcache_client.get(memcache_key)
+        if cached_channels is not None:
+            try:
+                channels = json.loads(cached_channels)
+            except JSONDecodeError:
+                log.exception(
+                    "Cannot load cached channels for %s; deleting" %
+                    search_host)
+                memcache_client.delete(memcache_key)
+        if (channels is None and
+                not getFeatureFlag(u"snap.disable_channel_search")):
+            path = "api/v1/channels"
+            timeline = cls._getTimeline()
+            if timeline is not None:
+                action = timeline.start("store-search-get", "/" + path)
+            channels_url = urlappend(config.snappy.store_search_url, path)
+            try:
+                response = urlfetch(
+                    channels_url, headers={"Accept": "application/hal+json"})
+            except requests.HTTPError as e:
+                raise BadSearchResponse(e.args[0])
+            finally:
+                if timeline is not None:
+                    action.finish()
+            channels = response.json().get("_embedded", {}).get(
+                "clickindex:channel", [])
+            expire_time = time.time() + 60 * 60 * 24
+            memcache_client.set(
+                memcache_key, json.dumps(channels), expire_time)
+        if channels is None:
+            channels = _default_store_channels
+        return channels
+
+    @classmethod
+    def release(cls, snapbuild, revision):
+        """See `ISnapStoreClient`."""
+        assert config.snappy.store_url is not None
+        snap = snapbuild.snap
+        assert snap.store_name is not None
+        assert snap.store_series is not None
+        assert snap.store_channels
+        release_url = urlappend(
+            config.snappy.store_url, "dev/api/snap-release/")
+        data = {
+            "name": snap.store_name,
+            "revision": revision,
+            # The security proxy is useless and breaks JSON serialisation.
+            "channels": removeSecurityProxy(snap.store_channels),
+            "series": snap.store_series.name,
+            }
+        # XXX cjwatson 2016-06-28: This should add timeline information, but
+        # that's currently difficult in jobs.
+        try:
+            assert snap.store_secrets is not None
+            urlfetch(
+                release_url, method="POST", json=data,
+                auth=MacaroonAuth(
+                    snap.store_secrets["root"],
+                    snap.store_secrets["discharge"]))
+        except requests.HTTPError as e:
+            if e.response is not None:
+                error = None
+                try:
+                    error = e.response.json()["errors"]
+                except Exception:
+                    pass
+                if error is not None:
+                    raise ReleaseFailedResponse(error)
+            raise BadReleaseResponse(e.args[0])

=== modified file 'lib/lp/snappy/tests/test_snapstoreclient.py'
--- lib/lp/snappy/tests/test_snapstoreclient.py	2016-06-27 13:19:15 +0000
+++ lib/lp/snappy/tests/test_snapstoreclient.py	2016-07-12 15:02:33 +0000
@@ -39,12 +39,16 @@
 from zope.component import getUtility
 
 from lp.services.features.testing import FeatureFixture
+from lp.services.memcache.interfaces import IMemcacheClient
 from lp.services.timeline.requesttimeline import get_request_timeline
 from lp.snappy.interfaces.snap import SNAP_TESTING_FLAGS
 from lp.snappy.interfaces.snapstoreclient import (
+    BadReleaseResponse,
     BadRequestPackageUploadResponse,
     BadScanStatusResponse,
+    BadSearchResponse,
     ISnapStoreClient,
+    ReleaseFailedResponse,
     ScanFailedResponse,
     UnauthorizedUploadResponse,
     UploadNotScannedYetResponse,
@@ -169,7 +173,8 @@
         self.useFixture(FeatureFixture(SNAP_TESTING_FLAGS))
         self.pushConfig(
             "snappy", store_url="http://sca.example/",
-            store_upload_url="http://updown.example/")
+            store_upload_url="http://updown.example/",
+            store_search_url="http://search.example/")
         self.pushConfig(
             "launchpad", openid_provider_root="http://sso.example/")
         self.client = getUtility(ISnapStoreClient)
@@ -225,6 +230,22 @@
             "content": {"discharge_macaroon": new_macaroon.serialize()},
             }
 
+    @urlmatch(path=r".*/snap-release/$")
+    def _snap_release_handler(self, url, request):
+        self.snap_release_request = request
+        return {
+            "status_code": 200,
+            "content": {
+                "success": True,
+                "channel_map": [
+                    {"channel": "stable", "info": "specific",
+                     "version": "1.0", "revision": 1},
+                    {"channel": "edge", "info": "specific",
+                     "version": "1.0", "revision": 1},
+                    ],
+                "opened_channels": ["stable", "edge"],
+                }}
+
     def test_requestPackageUploadPermission(self):
         @all_requests
         def handler(url, request):
@@ -452,7 +473,7 @@
         status_url = "http://sca.example/dev/api/click-scan-complete/updown/1/"
         with HTTMock(handler):
             self.assertEqual(
-                "http://sca.example/dev/click-apps/1/",
+                ("http://sca.example/dev/click-apps/1/", 1),
                 self.client.checkStatus(status_url))
 
     def test_checkStatus_new_pending(self):
@@ -525,7 +546,7 @@
         status_url = "http://sca.example/dev/api/snaps/1/builds/1/status"
         with HTTMock(handler):
             self.assertEqual(
-                "http://sca.example/dev/click-apps/1/rev/1/",
+                ("http://sca.example/dev/click-apps/1/rev/1/", 1),
                 self.client.checkStatus(status_url))
 
     def test_checkStatus_404(self):
@@ -538,3 +559,121 @@
             self.assertRaisesWithContent(
                 BadScanStatusResponse, b"404 Client Error: Not found",
                 self.client.checkStatus, status_url)
+
+    def test_listChannels(self):
+        expected_channels = [
+            {"name": "stable", "display_name": "Stable"},
+            {"name": "edge", "display_name": "Edge"},
+            ]
+
+        @all_requests
+        def handler(url, request):
+            self.request = request
+            return {
+                "status_code": 200,
+                "content": {
+                    "_embedded": {"clickindex:channel": expected_channels}}}
+
+        memcache_key = "search.example:channels".encode("UTF-8")
+        try:
+            with HTTMock(handler):
+                self.assertEqual(expected_channels, self.client.listChannels())
+            self.assertThat(self.request, RequestMatches(
+                url=Equals("http://search.example/api/v1/channels"),
+                method=Equals("GET"),
+                headers=ContainsDict(
+                    {"Accept": Equals("application/hal+json")})))
+            self.assertEqual(
+                expected_channels,
+                json.loads(getUtility(IMemcacheClient).get(memcache_key)))
+            self.request = None
+            with HTTMock(handler):
+                self.assertEqual(expected_channels, self.client.listChannels())
+            self.assertIsNone(self.request)
+        finally:
+            getUtility(IMemcacheClient).delete(memcache_key)
+
+    def test_listChannels_404(self):
+        @all_requests
+        def handler(url, request):
+            return {"status_code": 404, "reason": b"Not found"}
+
+        with HTTMock(handler):
+            self.assertRaisesWithContent(
+                BadSearchResponse, b"404 Client Error: Not found",
+                self.client.listChannels)
+
+    def test_listChannels_disable_search(self):
+        @all_requests
+        def handler(url, request):
+            self.request = request
+            return {"status_code": 404, "reason": b"Not found"}
+
+        self.useFixture(
+            FeatureFixture({u"snap.disable_channel_search": u"on"}))
+        expected_channels = [
+            {"name": "candidate", "display_name": "Candidate"},
+            {"name": "edge", "display_name": "Edge"},
+            {"name": "beta", "display_name": "Beta"},
+            {"name": "stable", "display_name": "Stable"},
+            ]
+        self.request = None
+        with HTTMock(handler):
+            self.assertEqual(expected_channels, self.client.listChannels())
+        self.assertIsNone(self.request)
+        memcache_key = "search.example:channels".encode("UTF-8")
+        self.assertIsNone(getUtility(IMemcacheClient).get(memcache_key))
+
+    def test_release(self):
+        snap = self.factory.makeSnap(
+            store_upload=True,
+            store_series=self.factory.makeSnappySeries(name="rolling"),
+            store_name="test-snap", store_secrets=self._make_store_secrets(),
+            store_channels=["stable", "edge"])
+        snapbuild = self.factory.makeSnapBuild(snap=snap)
+        with HTTMock(self._snap_release_handler):
+            self.client.release(snapbuild, 1)
+        self.assertThat(self.snap_release_request, RequestMatches(
+            url=Equals("http://sca.example/dev/api/snap-release/"),
+            method=Equals("POST"),
+            headers=ContainsDict({"Content-Type": Equals("application/json")}),
+            auth=("Macaroon", MacaroonsVerify(self.root_key)),
+            json_data={
+                "name": "test-snap", "revision": 1,
+                "channels": ["stable", "edge"], "series": "rolling",
+                }))
+
+    def test_release_error(self):
+        @urlmatch(path=r".*/snap-release/$")
+        def handler(url, request):
+            return {
+                "status_code": 503,
+                "content": {"success": False, "errors": "Failed to publish"},
+                }
+
+        snap = self.factory.makeSnap(
+            store_upload=True,
+            store_series=self.factory.makeSnappySeries(name="rolling"),
+            store_name="test-snap", store_secrets=self._make_store_secrets(),
+            store_channels=["stable", "edge"])
+        snapbuild = self.factory.makeSnapBuild(snap=snap)
+        with HTTMock(handler):
+            self.assertRaisesWithContent(
+                ReleaseFailedResponse, "Failed to publish",
+                self.client.release, snapbuild, 1)
+
+    def test_release_404(self):
+        @urlmatch(path=r".*/snap-release/$")
+        def handler(url, request):
+            return {"status_code": 404, "reason": b"Not found"}
+
+        snap = self.factory.makeSnap(
+            store_upload=True,
+            store_series=self.factory.makeSnappySeries(name="rolling"),
+            store_name="test-snap", store_secrets=self._make_store_secrets(),
+            store_channels=["stable", "edge"])
+        snapbuild = self.factory.makeSnapBuild(snap=snap)
+        with HTTMock(handler):
+            self.assertRaisesWithContent(
+                BadReleaseResponse, b"404 Client Error: Not found",
+                self.client.release, snapbuild, 1)

=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py	2016-07-11 22:35:59 +0000
+++ lib/lp/testing/factory.py	2016-07-12 15:02:33 +0000
@@ -4634,7 +4634,8 @@
                  auto_build_archive=None, auto_build_pocket=None,
                  is_stale=None, require_virtualized=True, processors=None,
                  date_created=DEFAULT, private=False, store_upload=False,
-                 store_series=None, store_name=None, store_secrets=None):
+                 store_series=None, store_name=None, store_secrets=None,
+                 store_channels=None):
         """Make a new Snap."""
         if registrant is None:
             registrant = self.makePerson()
@@ -4659,7 +4660,8 @@
             auto_build=auto_build, auto_build_archive=auto_build_archive,
             auto_build_pocket=auto_build_pocket, private=private,
             store_upload=store_upload, store_series=store_series,
-            store_name=store_name, store_secrets=store_secrets)
+            store_name=store_name, store_secrets=store_secrets,
+            store_channels=store_channels)
         if is_stale is not None:
             removeSecurityProxy(snap).is_stale = is_stale
         IStore(snap).flush()