Merge lp:~cjwatson/launchpad/snap-channels-store-client into lp:launchpad

Proposed by Colin Watson on 2016-06-30
Status: Merged
Merged at revision: 18142
Proposed branch: lp:~cjwatson/launchpad/snap-channels-store-client
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/snap-auto-build-ui
Diff against target: 565 lines (+320/-12) (has conflicts)
8 files modified
configs/development/launchpad-lazr.conf (+1/-0)
lib/lp/services/config/schema-lazr.conf (+3/-0)
lib/lp/snappy/interfaces/snap.py (+7/-0)
lib/lp/snappy/interfaces/snapstoreclient.py (+37/-2)
lib/lp/snappy/model/snap.py (+6/-3)
lib/lp/snappy/model/snapstoreclient.py (+120/-2)
lib/lp/snappy/tests/test_snapstoreclient.py (+142/-3)
lib/lp/testing/factory.py (+4/-2)
Text conflict in lib/lp/snappy/browser/snap.py
Text conflict in lib/lp/snappy/browser/widgets/tests/test_snaparchivewidget.py
To merge this branch: bzr merge lp:~cjwatson/launchpad/snap-channels-store-client
Reviewer Review Type Date Requested Status
Thomi Richards (community) Approve on 2016-07-13
Launchpad code reviewers 2016-06-30 Pending
Review via email: mp+298806@code.launchpad.net

Commit message

Add model and store client support for automatically releasing snap packages.

Description of the change

Add model and store client support for automatically releasing snap packages.

To post a comment you must log in.

LGTM - one comment below for your consideration.

Thanks for the fix - looks great.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'configs/development/launchpad-lazr.conf'
2--- configs/development/launchpad-lazr.conf 2016-05-18 00:33:18 +0000
3+++ configs/development/launchpad-lazr.conf 2016-07-12 15:02:33 +0000
4@@ -190,6 +190,7 @@
5 builder_proxy_auth_api_endpoint: http://snap-proxy.launchpad.dev:8080/tokens
6 builder_proxy_host: snap-proxy.launchpad.dev
7 builder_proxy_port: 3128
8+store_search_url: https://search.apps.ubuntu.com/
9 tools_source: deb http://ppa.launchpad.net/snappy-dev/snapcraft-daily/ubuntu %(series)s main
10
11 [txlongpoll]
12
13=== modified file 'lib/lp/services/config/schema-lazr.conf'
14--- lib/lp/services/config/schema-lazr.conf 2016-06-20 20:18:11 +0000
15+++ lib/lp/services/config/schema-lazr.conf 2016-07-12 15:02:33 +0000
16@@ -1804,6 +1804,9 @@
17 # The store's upload URL endpoint.
18 store_upload_url: none
19
20+# The store's search URL endpoint.
21+store_search_url: none
22+
23 [process-job-source-groups]
24 # This section is used by cronscripts/process-job-source-groups.py.
25 dbuser: process-job-source-groups
26
27=== modified file 'lib/lp/snappy/interfaces/snap.py'
28--- lib/lp/snappy/interfaces/snap.py 2016-07-05 10:20:51 +0000
29+++ lib/lp/snappy/interfaces/snap.py 2016-07-12 15:02:33 +0000
30@@ -421,6 +421,13 @@
31 "Serialized secrets issued by the store and the login service to "
32 "authorize uploads of this snap package."))
33
34+ store_channels = List(
35+ value_type=TextLine(), title=_("Store channels"),
36+ required=False, readonly=False,
37+ description=_(
38+ "Channels to release this snap package to after uploading it to "
39+ "the store."))
40+
41
42 class ISnapAdminAttributes(Interface):
43 """`ISnap` attributes that can be edited by admins.
44
45=== modified file 'lib/lp/snappy/interfaces/snapstoreclient.py'
46--- lib/lp/snappy/interfaces/snapstoreclient.py 2016-06-21 14:51:06 +0000
47+++ lib/lp/snappy/interfaces/snapstoreclient.py 2016-07-12 15:02:33 +0000
48@@ -8,11 +8,14 @@
49 __metaclass__ = type
50 __all__ = [
51 'BadRefreshResponse',
52+ 'BadReleaseResponse',
53 'BadRequestPackageUploadResponse',
54 'BadScanStatusResponse',
55+ 'BadSearchResponse',
56 'BadUploadResponse',
57 'ISnapStoreClient',
58 'NeedsRefreshResponse',
59+ 'ReleaseFailedResponse',
60 'ScanFailedResponse',
61 'UnauthorizedUploadResponse',
62 'UploadNotScannedYetResponse',
63@@ -53,6 +56,18 @@
64 pass
65
66
67+class BadSearchResponse(Exception):
68+ pass
69+
70+
71+class BadReleaseResponse(Exception):
72+ pass
73+
74+
75+class ReleaseFailedResponse(Exception):
76+ pass
77+
78+
79 class ISnapStoreClient(Interface):
80 """Interface for the API provided by the snap store."""
81
82@@ -91,6 +106,26 @@
83 scanned the upload.
84 :raises BadScanStatusResponse: if the store failed to scan the
85 upload.
86- :return: A URL on the store with further information about this
87- upload.
88+ :return: A tuple of (`url`, `revision`), where `url` is a URL on the
89+ store with further information about this upload, and `revision`
90+ is the store revision number for the upload or None.
91+ """
92+
93+ def listChannels():
94+ """Fetch the current list of channels from the store.
95+
96+ :raises BadSearchResponse: if the attempt to fetch the list of
97+ channels from the store fails.
98+ :return: A list of dictionaries, one per channel, each of which
99+ contains at least "name" and "display_name" keys.
100+ """
101+
102+ def release(snapbuild, revision):
103+ """Tell the store to release a snap build to specified channels.
104+
105+ :param snapbuild: The `ISnapBuild` to release.
106+ :param revision: The revision returned by the store when uploading
107+ the build.
108+ :raises BadReleaseResponse: if the store failed to release the
109+ build.
110 """
111
112=== modified file 'lib/lp/snappy/model/snap.py'
113--- lib/lp/snappy/model/snap.py 2016-07-05 10:20:51 +0000
114+++ lib/lp/snappy/model/snap.py 2016-07-12 15:02:33 +0000
115@@ -185,12 +185,14 @@
116
117 store_secrets = JSON('store_secrets', allow_none=True)
118
119+ store_channels = JSON('store_channels', allow_none=True)
120+
121 def __init__(self, registrant, owner, distro_series, name,
122 description=None, branch=None, git_ref=None, auto_build=False,
123 auto_build_archive=None, auto_build_pocket=None,
124 require_virtualized=True, date_created=DEFAULT,
125 private=False, store_upload=False, store_series=None,
126- store_name=None, store_secrets=None):
127+ store_name=None, store_secrets=None, store_channels=None):
128 """Construct a `Snap`."""
129 super(Snap, self).__init__()
130 self.registrant = registrant
131@@ -211,6 +213,7 @@
132 self.store_series = store_series
133 self.store_name = store_name
134 self.store_secrets = store_secrets
135+ self.store_channels = store_channels
136
137 @property
138 def valid_webhook_event_types(self):
139@@ -506,7 +509,7 @@
140 auto_build_archive=None, auto_build_pocket=None,
141 require_virtualized=True, processors=None, date_created=DEFAULT,
142 private=False, store_upload=False, store_series=None,
143- store_name=None, store_secrets=None):
144+ store_name=None, store_secrets=None, store_channels=None):
145 """See `ISnapSet`."""
146 if not registrant.inTeam(owner):
147 if owner.is_team:
148@@ -535,7 +538,7 @@
149 require_virtualized=require_virtualized, date_created=date_created,
150 private=private, store_upload=store_upload,
151 store_series=store_series, store_name=store_name,
152- store_secrets=store_secrets)
153+ store_secrets=store_secrets, store_channels=store_channels)
154 store.add(snap)
155
156 if processors is None:
157
158=== modified file 'lib/lp/snappy/model/snapstoreclient.py'
159--- lib/lp/snappy/model/snapstoreclient.py 2016-06-27 13:19:15 +0000
160+++ lib/lp/snappy/model/snapstoreclient.py 2016-07-12 15:02:33 +0000
161@@ -10,25 +10,40 @@
162 'SnapStoreClient',
163 ]
164
165+import json
166+try:
167+ from json.decoder import JSONDecodeError
168+except ImportError:
169+ JSONDecodeError = ValueError
170 import string
171+import time
172+from urlparse import urlsplit
173
174 from lazr.restful.utils import get_current_browser_request
175 from pymacaroons import Macaroon
176 import requests
177 from requests_toolbelt import MultipartEncoder
178+from zope.component import getUtility
179 from zope.interface import implementer
180+from zope.security.proxy import removeSecurityProxy
181
182 from lp.services.config import config
183+from lp.services.features import getFeatureFlag
184+from lp.services.memcache.interfaces import IMemcacheClient
185+from lp.services.scripts import log
186 from lp.services.timeline.requesttimeline import get_request_timeline
187 from lp.services.timeout import urlfetch
188 from lp.services.webapp.url import urlappend
189 from lp.snappy.interfaces.snapstoreclient import (
190 BadRefreshResponse,
191+ BadReleaseResponse,
192 BadRequestPackageUploadResponse,
193 BadScanStatusResponse,
194+ BadSearchResponse,
195 BadUploadResponse,
196 ISnapStoreClient,
197 NeedsRefreshResponse,
198+ ReleaseFailedResponse,
199 ScanFailedResponse,
200 UnauthorizedUploadResponse,
201 UploadNotScannedYetResponse,
202@@ -99,10 +114,28 @@
203 return r
204
205
206+# Hardcoded fallback.
207+_default_store_channels = [
208+ {"name": "candidate", "display_name": "Candidate"},
209+ {"name": "edge", "display_name": "Edge"},
210+ {"name": "beta", "display_name": "Beta"},
211+ {"name": "stable", "display_name": "Stable"},
212+ ]
213+
214+
215 @implementer(ISnapStoreClient)
216 class SnapStoreClient:
217 """A client for the API provided by the snap store."""
218
219+ @staticmethod
220+ def _getTimeline():
221+ # XXX cjwatson 2016-06-29: This can be simplified once jobs have
222+ # timeline support.
223+ request = get_current_browser_request()
224+ if request is None:
225+ return None
226+ return get_request_timeline(request)
227+
228 def requestPackageUploadPermission(self, snappy_series, snap_name):
229 assert config.snappy.store_url is not None
230 request_url = urlappend(config.snappy.store_url, "dev/api/acl/")
231@@ -203,6 +236,7 @@
232
233 @classmethod
234 def refreshDischargeMacaroon(cls, snap):
235+ """See `ISnapStoreClient`."""
236 assert config.launchpad.openid_provider_root is not None
237 assert snap.store_secrets is not None
238 refresh_url = urlappend(
239@@ -222,6 +256,7 @@
240
241 @classmethod
242 def checkStatus(cls, status_url):
243+ """See `ISnapStoreClient`."""
244 try:
245 response = urlfetch(status_url)
246 response_data = response.json()
247@@ -232,7 +267,9 @@
248 elif not response_data["application_url"]:
249 raise ScanFailedResponse(response_data["message"])
250 else:
251- return response_data["application_url"]
252+ return (
253+ response_data["application_url"],
254+ response_data["revision"])
255 else:
256 # New status format.
257 if not response_data["processed"]:
258@@ -241,7 +278,88 @@
259 error_message = "\n".join(
260 error["message"] for error in response_data["errors"])
261 raise ScanFailedResponse(error_message)
262+ elif not response_data["can_release"]:
263+ return response_data["url"], None
264 else:
265- return response_data["url"]
266+ return response_data["url"], response_data["revision"]
267 except requests.HTTPError as e:
268 raise BadScanStatusResponse(e.args[0])
269+
270+ @classmethod
271+ def listChannels(cls):
272+ """See `ISnapStoreClient`."""
273+ if config.snappy.store_search_url is None:
274+ return _default_store_channels
275+ channels = None
276+ memcache_client = getUtility(IMemcacheClient)
277+ search_host = urlsplit(config.snappy.store_search_url).hostname
278+ memcache_key = ("%s:channels" % search_host).encode("UTF-8")
279+ cached_channels = memcache_client.get(memcache_key)
280+ if cached_channels is not None:
281+ try:
282+ channels = json.loads(cached_channels)
283+ except JSONDecodeError:
284+ log.exception(
285+ "Cannot load cached channels for %s; deleting" %
286+ search_host)
287+ memcache_client.delete(memcache_key)
288+ if (channels is None and
289+ not getFeatureFlag(u"snap.disable_channel_search")):
290+ path = "api/v1/channels"
291+ timeline = cls._getTimeline()
292+ if timeline is not None:
293+ action = timeline.start("store-search-get", "/" + path)
294+ channels_url = urlappend(config.snappy.store_search_url, path)
295+ try:
296+ response = urlfetch(
297+ channels_url, headers={"Accept": "application/hal+json"})
298+ except requests.HTTPError as e:
299+ raise BadSearchResponse(e.args[0])
300+ finally:
301+ if timeline is not None:
302+ action.finish()
303+ channels = response.json().get("_embedded", {}).get(
304+ "clickindex:channel", [])
305+ expire_time = time.time() + 60 * 60 * 24
306+ memcache_client.set(
307+ memcache_key, json.dumps(channels), expire_time)
308+ if channels is None:
309+ channels = _default_store_channels
310+ return channels
311+
312+ @classmethod
313+ def release(cls, snapbuild, revision):
314+ """See `ISnapStoreClient`."""
315+ assert config.snappy.store_url is not None
316+ snap = snapbuild.snap
317+ assert snap.store_name is not None
318+ assert snap.store_series is not None
319+ assert snap.store_channels
320+ release_url = urlappend(
321+ config.snappy.store_url, "dev/api/snap-release/")
322+ data = {
323+ "name": snap.store_name,
324+ "revision": revision,
325+ # The security proxy is useless and breaks JSON serialisation.
326+ "channels": removeSecurityProxy(snap.store_channels),
327+ "series": snap.store_series.name,
328+ }
329+ # XXX cjwatson 2016-06-28: This should add timeline information, but
330+ # that's currently difficult in jobs.
331+ try:
332+ assert snap.store_secrets is not None
333+ urlfetch(
334+ release_url, method="POST", json=data,
335+ auth=MacaroonAuth(
336+ snap.store_secrets["root"],
337+ snap.store_secrets["discharge"]))
338+ except requests.HTTPError as e:
339+ if e.response is not None:
340+ error = None
341+ try:
342+ error = e.response.json()["errors"]
343+ except Exception:
344+ pass
345+ if error is not None:
346+ raise ReleaseFailedResponse(error)
347+ raise BadReleaseResponse(e.args[0])
348
349=== modified file 'lib/lp/snappy/tests/test_snapstoreclient.py'
350--- lib/lp/snappy/tests/test_snapstoreclient.py 2016-06-27 13:19:15 +0000
351+++ lib/lp/snappy/tests/test_snapstoreclient.py 2016-07-12 15:02:33 +0000
352@@ -39,12 +39,16 @@
353 from zope.component import getUtility
354
355 from lp.services.features.testing import FeatureFixture
356+from lp.services.memcache.interfaces import IMemcacheClient
357 from lp.services.timeline.requesttimeline import get_request_timeline
358 from lp.snappy.interfaces.snap import SNAP_TESTING_FLAGS
359 from lp.snappy.interfaces.snapstoreclient import (
360+ BadReleaseResponse,
361 BadRequestPackageUploadResponse,
362 BadScanStatusResponse,
363+ BadSearchResponse,
364 ISnapStoreClient,
365+ ReleaseFailedResponse,
366 ScanFailedResponse,
367 UnauthorizedUploadResponse,
368 UploadNotScannedYetResponse,
369@@ -169,7 +173,8 @@
370 self.useFixture(FeatureFixture(SNAP_TESTING_FLAGS))
371 self.pushConfig(
372 "snappy", store_url="http://sca.example/",
373- store_upload_url="http://updown.example/")
374+ store_upload_url="http://updown.example/",
375+ store_search_url="http://search.example/")
376 self.pushConfig(
377 "launchpad", openid_provider_root="http://sso.example/")
378 self.client = getUtility(ISnapStoreClient)
379@@ -225,6 +230,22 @@
380 "content": {"discharge_macaroon": new_macaroon.serialize()},
381 }
382
383+ @urlmatch(path=r".*/snap-release/$")
384+ def _snap_release_handler(self, url, request):
385+ self.snap_release_request = request
386+ return {
387+ "status_code": 200,
388+ "content": {
389+ "success": True,
390+ "channel_map": [
391+ {"channel": "stable", "info": "specific",
392+ "version": "1.0", "revision": 1},
393+ {"channel": "edge", "info": "specific",
394+ "version": "1.0", "revision": 1},
395+ ],
396+ "opened_channels": ["stable", "edge"],
397+ }}
398+
399 def test_requestPackageUploadPermission(self):
400 @all_requests
401 def handler(url, request):
402@@ -452,7 +473,7 @@
403 status_url = "http://sca.example/dev/api/click-scan-complete/updown/1/"
404 with HTTMock(handler):
405 self.assertEqual(
406- "http://sca.example/dev/click-apps/1/",
407+ ("http://sca.example/dev/click-apps/1/", 1),
408 self.client.checkStatus(status_url))
409
410 def test_checkStatus_new_pending(self):
411@@ -525,7 +546,7 @@
412 status_url = "http://sca.example/dev/api/snaps/1/builds/1/status"
413 with HTTMock(handler):
414 self.assertEqual(
415- "http://sca.example/dev/click-apps/1/rev/1/",
416+ ("http://sca.example/dev/click-apps/1/rev/1/", 1),
417 self.client.checkStatus(status_url))
418
419 def test_checkStatus_404(self):
420@@ -538,3 +559,121 @@
421 self.assertRaisesWithContent(
422 BadScanStatusResponse, b"404 Client Error: Not found",
423 self.client.checkStatus, status_url)
424+
425+ def test_listChannels(self):
426+ expected_channels = [
427+ {"name": "stable", "display_name": "Stable"},
428+ {"name": "edge", "display_name": "Edge"},
429+ ]
430+
431+ @all_requests
432+ def handler(url, request):
433+ self.request = request
434+ return {
435+ "status_code": 200,
436+ "content": {
437+ "_embedded": {"clickindex:channel": expected_channels}}}
438+
439+ memcache_key = "search.example:channels".encode("UTF-8")
440+ try:
441+ with HTTMock(handler):
442+ self.assertEqual(expected_channels, self.client.listChannels())
443+ self.assertThat(self.request, RequestMatches(
444+ url=Equals("http://search.example/api/v1/channels"),
445+ method=Equals("GET"),
446+ headers=ContainsDict(
447+ {"Accept": Equals("application/hal+json")})))
448+ self.assertEqual(
449+ expected_channels,
450+ json.loads(getUtility(IMemcacheClient).get(memcache_key)))
451+ self.request = None
452+ with HTTMock(handler):
453+ self.assertEqual(expected_channels, self.client.listChannels())
454+ self.assertIsNone(self.request)
455+ finally:
456+ getUtility(IMemcacheClient).delete(memcache_key)
457+
458+ def test_listChannels_404(self):
459+ @all_requests
460+ def handler(url, request):
461+ return {"status_code": 404, "reason": b"Not found"}
462+
463+ with HTTMock(handler):
464+ self.assertRaisesWithContent(
465+ BadSearchResponse, b"404 Client Error: Not found",
466+ self.client.listChannels)
467+
468+ def test_listChannels_disable_search(self):
469+ @all_requests
470+ def handler(url, request):
471+ self.request = request
472+ return {"status_code": 404, "reason": b"Not found"}
473+
474+ self.useFixture(
475+ FeatureFixture({u"snap.disable_channel_search": u"on"}))
476+ expected_channels = [
477+ {"name": "candidate", "display_name": "Candidate"},
478+ {"name": "edge", "display_name": "Edge"},
479+ {"name": "beta", "display_name": "Beta"},
480+ {"name": "stable", "display_name": "Stable"},
481+ ]
482+ self.request = None
483+ with HTTMock(handler):
484+ self.assertEqual(expected_channels, self.client.listChannels())
485+ self.assertIsNone(self.request)
486+ memcache_key = "search.example:channels".encode("UTF-8")
487+ self.assertIsNone(getUtility(IMemcacheClient).get(memcache_key))
488+
489+ def test_release(self):
490+ snap = self.factory.makeSnap(
491+ store_upload=True,
492+ store_series=self.factory.makeSnappySeries(name="rolling"),
493+ store_name="test-snap", store_secrets=self._make_store_secrets(),
494+ store_channels=["stable", "edge"])
495+ snapbuild = self.factory.makeSnapBuild(snap=snap)
496+ with HTTMock(self._snap_release_handler):
497+ self.client.release(snapbuild, 1)
498+ self.assertThat(self.snap_release_request, RequestMatches(
499+ url=Equals("http://sca.example/dev/api/snap-release/"),
500+ method=Equals("POST"),
501+ headers=ContainsDict({"Content-Type": Equals("application/json")}),
502+ auth=("Macaroon", MacaroonsVerify(self.root_key)),
503+ json_data={
504+ "name": "test-snap", "revision": 1,
505+ "channels": ["stable", "edge"], "series": "rolling",
506+ }))
507+
508+ def test_release_error(self):
509+ @urlmatch(path=r".*/snap-release/$")
510+ def handler(url, request):
511+ return {
512+ "status_code": 503,
513+ "content": {"success": False, "errors": "Failed to publish"},
514+ }
515+
516+ snap = self.factory.makeSnap(
517+ store_upload=True,
518+ store_series=self.factory.makeSnappySeries(name="rolling"),
519+ store_name="test-snap", store_secrets=self._make_store_secrets(),
520+ store_channels=["stable", "edge"])
521+ snapbuild = self.factory.makeSnapBuild(snap=snap)
522+ with HTTMock(handler):
523+ self.assertRaisesWithContent(
524+ ReleaseFailedResponse, "Failed to publish",
525+ self.client.release, snapbuild, 1)
526+
527+ def test_release_404(self):
528+ @urlmatch(path=r".*/snap-release/$")
529+ def handler(url, request):
530+ return {"status_code": 404, "reason": b"Not found"}
531+
532+ snap = self.factory.makeSnap(
533+ store_upload=True,
534+ store_series=self.factory.makeSnappySeries(name="rolling"),
535+ store_name="test-snap", store_secrets=self._make_store_secrets(),
536+ store_channels=["stable", "edge"])
537+ snapbuild = self.factory.makeSnapBuild(snap=snap)
538+ with HTTMock(handler):
539+ self.assertRaisesWithContent(
540+ BadReleaseResponse, b"404 Client Error: Not found",
541+ self.client.release, snapbuild, 1)
542
543=== modified file 'lib/lp/testing/factory.py'
544--- lib/lp/testing/factory.py 2016-07-11 22:35:59 +0000
545+++ lib/lp/testing/factory.py 2016-07-12 15:02:33 +0000
546@@ -4634,7 +4634,8 @@
547 auto_build_archive=None, auto_build_pocket=None,
548 is_stale=None, require_virtualized=True, processors=None,
549 date_created=DEFAULT, private=False, store_upload=False,
550- store_series=None, store_name=None, store_secrets=None):
551+ store_series=None, store_name=None, store_secrets=None,
552+ store_channels=None):
553 """Make a new Snap."""
554 if registrant is None:
555 registrant = self.makePerson()
556@@ -4659,7 +4660,8 @@
557 auto_build=auto_build, auto_build_archive=auto_build_archive,
558 auto_build_pocket=auto_build_pocket, private=private,
559 store_upload=store_upload, store_series=store_series,
560- store_name=store_name, store_secrets=store_secrets)
561+ store_name=store_name, store_secrets=store_secrets,
562+ store_channels=store_channels)
563 if is_stale is not None:
564 removeSecurityProxy(snap).is_stale = is_stale
565 IStore(snap).flush()