Merge lp:~cjwatson/launchpad/snap-store-client into lp:launchpad

Proposed by Colin Watson
Status: Merged
Merged at revision: 18031
Proposed branch: lp:~cjwatson/launchpad/snap-store-client
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/snap-upload-model
Diff against target: 545 lines (+451/-1)
9 files modified
lib/lp/services/config/schema-lazr.conf (+6/-0)
lib/lp/snappy/configure.zcml (+7/-0)
lib/lp/snappy/interfaces/snap.py (+4/-0)
lib/lp/snappy/interfaces/snapstoreclient.py (+47/-0)
lib/lp/snappy/model/snap.py (+12/-0)
lib/lp/snappy/model/snapstoreclient.py (+159/-0)
lib/lp/snappy/tests/test_snapstoreclient.py (+213/-0)
setup.py (+2/-1)
versions.cfg (+1/-0)
To merge this branch: bzr merge lp:~cjwatson/launchpad/snap-store-client
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+293668@code.launchpad.net

Commit message

Add an initial client for uploading snaps to the store.

Description of the change

Add an initial client for uploading snaps to the store.

This isn't used anywhere yet, and the implementation will need to change to handle recently-agreed changes to how discharge macaroons are handled, but it's useful to have an outline in place.

To post a comment you must log in.
William Grant (wgrant) :
review: Approve (code)
Colin Watson (cjwatson) :

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/services/config/schema-lazr.conf'
2--- lib/lp/services/config/schema-lazr.conf 2016-03-31 14:07:33 +0000
3+++ lib/lp/services/config/schema-lazr.conf 2016-05-12 10:54:06 +0000
4@@ -1779,6 +1779,12 @@
5 # datatype: string
6 tools_source: none
7
8+# The store's primary URL endpoint.
9+store_url: none
10+
11+# The store's upload URL endpoint.
12+store_upload_url: none
13+
14 [process-job-source-groups]
15 # This section is used by cronscripts/process-job-source-groups.py.
16 dbuser: process-job-source-groups
17
18=== modified file 'lib/lp/snappy/configure.zcml'
19--- lib/lp/snappy/configure.zcml 2016-05-05 20:02:01 +0000
20+++ lib/lp/snappy/configure.zcml 2016-05-12 10:54:06 +0000
21@@ -117,6 +117,13 @@
22 interface="lp.snappy.interfaces.snappyseries.ISnappyDistroSeriesSet" />
23 </securedutility>
24
25+ <!-- Store interaction -->
26+ <securedutility
27+ class="lp.snappy.model.snapstoreclient.SnapStoreClient"
28+ provides="lp.snappy.interfaces.snapstoreclient.ISnapStoreClient">
29+ <allow interface="lp.snappy.interfaces.snapstoreclient.ISnapStoreClient" />
30+ </securedutility>
31+
32 <webservice:register module="lp.snappy.interfaces.webservice" />
33
34 </configure>
35
36=== modified file 'lib/lp/snappy/interfaces/snap.py'
37--- lib/lp/snappy/interfaces/snap.py 2016-05-06 12:46:46 +0000
38+++ lib/lp/snappy/interfaces/snap.py 2016-05-12 10:54:06 +0000
39@@ -249,6 +249,10 @@
40 :return: Sequence of `IDistroArchSeries` instances.
41 """
42
43+ can_upload_to_store = Attribute(
44+ "Whether everything is set up to allow uploading builds of this snap "
45+ "package to the store.")
46+
47 @call_with(requester=REQUEST_USER)
48 @operation_parameters(
49 archive=Reference(schema=IArchive),
50
51=== added file 'lib/lp/snappy/interfaces/snapstoreclient.py'
52--- lib/lp/snappy/interfaces/snapstoreclient.py 1970-01-01 00:00:00 +0000
53+++ lib/lp/snappy/interfaces/snapstoreclient.py 2016-05-12 10:54:06 +0000
54@@ -0,0 +1,47 @@
55+# Copyright 2016 Canonical Ltd. This software is licensed under the
56+# GNU Affero General Public License version 3 (see the file LICENSE).
57+
58+"""Interface for communication with the snap store."""
59+
60+from __future__ import absolute_import, print_function, unicode_literals
61+
62+__metaclass__ = type
63+__all__ = [
64+ 'BadRequestPackageUploadResponse',
65+ 'BadUploadResponse',
66+ 'ISnapStoreClient',
67+ ]
68+
69+from zope.interface import Interface
70+
71+
72+class BadRequestPackageUploadResponse(Exception):
73+ pass
74+
75+
76+class BadUploadResponse(Exception):
77+ pass
78+
79+
80+class ISnapStoreClient(Interface):
81+ """Interface for the API provided by the snap store."""
82+
83+ def requestPackageUploadPermission(snappy_series, snap_name):
84+ """Request permission from the store to upload builds of a snap.
85+
86+ The returned macaroon will include a third-party caveat that must be
87+ discharged by the login service. This method does not acquire that
88+ discharge; it must be acquired separately.
89+
90+ :param snappy_series: The `ISnappySeries` in which this snap should
91+ be published on the store.
92+ :param snap_name: The registered name of this snap on the store.
93+ :return: A serialized macaroon appropriate for uploading builds of
94+ this snap.
95+ """
96+
97+ def upload(snapbuild):
98+ """Upload a snap build to the store.
99+
100+ :param snapbuild: The `ISnapBuild` to upload.
101+ """
102
103=== modified file 'lib/lp/snappy/model/snap.py'
104--- lib/lp/snappy/model/snap.py 2016-05-06 12:46:46 +0000
105+++ lib/lp/snappy/model/snap.py 2016-05-12 10:54:06 +0000
106@@ -57,6 +57,7 @@
107 IHasOwner,
108 IPersonRoles,
109 )
110+from lp.services.config import config
111 from lp.services.database.bulk import load_related
112 from lp.services.database.constants import (
113 DEFAULT,
114@@ -281,6 +282,17 @@
115 das.processor.supports_virtualized
116 or not self.require_virtualized))]
117
118+ @property
119+ def can_upload_to_store(self):
120+ return (
121+ config.snappy.store_upload_url is not None and
122+ config.snappy.store_url is not None and
123+ self.store_upload and
124+ self.store_series is not None and
125+ self.store_name is not None and
126+ self.store_secrets is not None and
127+ "discharge" in self.store_secrets)
128+
129 def requestBuild(self, requester, archive, distro_arch_series, pocket):
130 """See `ISnap`."""
131 if not requester.inTeam(self.owner):
132
133=== added file 'lib/lp/snappy/model/snapstoreclient.py'
134--- lib/lp/snappy/model/snapstoreclient.py 1970-01-01 00:00:00 +0000
135+++ lib/lp/snappy/model/snapstoreclient.py 2016-05-12 10:54:06 +0000
136@@ -0,0 +1,159 @@
137+# Copyright 2016 Canonical Ltd. This software is licensed under the
138+# GNU Affero General Public License version 3 (see the file LICENSE).
139+
140+"""Communication with the snap store."""
141+
142+from __future__ import absolute_import, print_function, unicode_literals
143+
144+__metaclass__ = type
145+__all__ = [
146+ 'SnapStoreClient',
147+ ]
148+
149+import string
150+try:
151+ from urllib.parse import quote_plus
152+except ImportError:
153+ from urllib import quote_plus
154+
155+from lazr.restful.utils import get_current_browser_request
156+import requests
157+from requests_toolbelt import MultipartEncoder
158+from zope.interface import implementer
159+
160+from lp.services.config import config
161+from lp.services.timeline.requesttimeline import get_request_timeline
162+from lp.services.timeout import urlfetch
163+from lp.services.webapp.url import urlappend
164+from lp.snappy.interfaces.snapstoreclient import (
165+ BadRequestPackageUploadResponse,
166+ BadUploadResponse,
167+ ISnapStoreClient,
168+ )
169+
170+
171+class LibraryFileAliasWrapper:
172+ """A `LibraryFileAlias` wrapper usable with a `MultipartEncoder`."""
173+
174+ def __init__(self, lfa):
175+ self.lfa = lfa
176+ self.position = 0
177+
178+ @property
179+ def len(self):
180+ return self.lfa.content.filesize - self.position
181+
182+ def read(self, length=-1):
183+ chunksize = None if length == -1 else length
184+ data = self.lfa.read(chunksize=chunksize)
185+ if chunksize is None:
186+ self.position = self.lfa.content.filesize
187+ else:
188+ self.position += length
189+ return data
190+
191+
192+class MacaroonAuth(requests.auth.AuthBase):
193+ """Attaches macaroon authentication to a given Request object."""
194+
195+ # The union of the base64 and URL-safe base64 alphabets.
196+ allowed_chars = set(string.digits + string.letters + "+/=-_")
197+
198+ def __init__(self, tokens):
199+ self.tokens = tokens
200+
201+ def __call__(self, r):
202+ params = []
203+ for k, v in self.tokens.items():
204+ # Check framing.
205+ assert set(k).issubset(self.allowed_chars)
206+ assert set(v).issubset(self.allowed_chars)
207+ params.append('%s="%s"' % (k, v))
208+ r.headers["Authorization"] = "Macaroon " + ", ".join(params)
209+ return r
210+
211+
212+@implementer(ISnapStoreClient)
213+class SnapStoreClient:
214+ """A client for the API provided by the snap store."""
215+
216+ def requestPackageUploadPermission(self, snappy_series, snap_name):
217+ assert config.snappy.store_url is not None
218+ request_url = urlappend(config.snappy.store_url, "dev/api/acl/")
219+ request = get_current_browser_request()
220+ timeline_action = get_request_timeline(request).start(
221+ "request-snap-upload-macaroon",
222+ "%s/%s" % (snappy_series.name, snap_name), allow_nested=True)
223+ try:
224+ response = urlfetch(
225+ request_url, method="POST",
226+ json={
227+ "packages": [
228+ {"name": snap_name, "series": snappy_series.name}],
229+ "permissions": ["package_upload"],
230+ })
231+ response_data = response.json()
232+ if "macaroon" not in response_data:
233+ raise BadRequestPackageUploadResponse(response.text)
234+ return response_data["macaroon"]
235+ except requests.HTTPError as e:
236+ raise BadRequestPackageUploadResponse(e.args[0])
237+ finally:
238+ timeline_action.finish()
239+
240+ def _uploadFile(self, lfa, lfc):
241+ """Upload a single file."""
242+ assert config.snappy.store_upload_url is not None
243+ unscanned_upload_url = urlappend(
244+ config.snappy.store_upload_url, "unscanned-upload/")
245+ lfa.open()
246+ try:
247+ lfa_wrapper = LibraryFileAliasWrapper(lfa)
248+ encoder = MultipartEncoder(
249+ fields={
250+ "binary": (
251+ "filename", lfa_wrapper, "application/octet-stream"),
252+ })
253+ # XXX cjwatson 2016-05-09: This should add timeline information,
254+ # but that's currently difficult in jobs.
255+ try:
256+ response = urlfetch(
257+ unscanned_upload_url, method="POST", data=encoder,
258+ headers={"Content-Type": encoder.content_type})
259+ response_data = response.json()
260+ if not response_data.get("successful", False):
261+ raise BadUploadResponse(response.text)
262+ return {"upload_id": response_data["upload_id"]}
263+ except requests.HTTPError as e:
264+ raise BadUploadResponse(e.args[0])
265+ finally:
266+ lfa.close()
267+
268+ def _uploadApp(self, snap, upload_data):
269+ """Create a new store upload based on the uploaded file."""
270+ assert config.snappy.store_url is not None
271+ assert snap.store_name is not None
272+ upload_url = urlappend(config.snappy.store_url, "dev/api/snap-upload/")
273+ data = {
274+ "name": snap.store_name,
275+ "updown_id": upload_data["upload_id"],
276+ "series": snap.store_series.name,
277+ }
278+ # XXX cjwatson 2016-04-20: handle refresh
279+ # XXX cjwatson 2016-05-09: This should add timeline information, but
280+ # that's currently difficult in jobs.
281+ try:
282+ assert snap.store_secrets is not None
283+ assert "discharge" in snap.store_secrets
284+ urlfetch(
285+ upload_url, method="POST", data=data,
286+ auth=MacaroonAuth(snap.store_secrets))
287+ except requests.HTTPError as e:
288+ raise BadUploadResponse(e.args[0])
289+
290+ def upload(self, snapbuild):
291+ """See `ISnapStoreClient`."""
292+ assert snapbuild.snap.can_upload_to_store
293+ for _, lfa, lfc in snapbuild.getFiles():
294+ upload_data = self._uploadFile(lfa, lfc)
295+ self._uploadApp(snapbuild.snap, upload_data)
296
297=== added file 'lib/lp/snappy/tests/test_snapstoreclient.py'
298--- lib/lp/snappy/tests/test_snapstoreclient.py 1970-01-01 00:00:00 +0000
299+++ lib/lp/snappy/tests/test_snapstoreclient.py 2016-05-12 10:54:06 +0000
300@@ -0,0 +1,213 @@
301+# Copyright 2016 Canonical Ltd. This software is licensed under the
302+# GNU Affero General Public License version 3 (see the file LICENSE).
303+
304+"""Tests for communication with the snap store."""
305+
306+from __future__ import absolute_import, print_function, unicode_literals
307+
308+__metaclass__ = type
309+
310+from cgi import FieldStorage
311+from collections import OrderedDict
312+import io
313+import json
314+
315+from httmock import (
316+ all_requests,
317+ HTTMock,
318+ urlmatch,
319+ )
320+from lazr.restful.utils import get_current_browser_request
321+from requests import Request
322+from requests.utils import parse_dict_header
323+from testtools.matchers import (
324+ Contains,
325+ Equals,
326+ Matcher,
327+ MatchesDict,
328+ MatchesStructure,
329+ StartsWith,
330+ )
331+import transaction
332+from zope.component import getUtility
333+
334+from lp.services.features.testing import FeatureFixture
335+from lp.services.timeline.requesttimeline import get_request_timeline
336+from lp.snappy.interfaces.snap import SNAP_TESTING_FLAGS
337+from lp.snappy.interfaces.snapstoreclient import (
338+ BadRequestPackageUploadResponse,
339+ ISnapStoreClient,
340+ )
341+from lp.snappy.model.snapstoreclient import MacaroonAuth
342+from lp.testing import (
343+ TestCase,
344+ TestCaseWithFactory,
345+ )
346+from lp.testing.layers import LaunchpadZopelessLayer
347+
348+
349+class TestMacaroonAuth(TestCase):
350+
351+ def test_good(self):
352+ r = Request()
353+ MacaroonAuth(OrderedDict([("root", "abc"), ("discharge", "def")]))(r)
354+ self.assertEqual(
355+ 'Macaroon root="abc", discharge="def"', r.headers["Authorization"])
356+
357+ def test_bad_framing(self):
358+ r = Request()
359+ self.assertRaises(AssertionError, MacaroonAuth({"root": 'ev"il'}), r)
360+
361+
362+class RequestMatches(Matcher):
363+ """Matches a request with the specified attributes."""
364+
365+ def __init__(self, url, auth=None, json_data=None, form_data=None,
366+ **kwargs):
367+ self.url = url
368+ self.auth = auth
369+ self.json_data = json_data
370+ self.form_data = form_data
371+ self.kwargs = kwargs
372+
373+ def match(self, request):
374+ mismatch = MatchesStructure(url=self.url, **self.kwargs).match(request)
375+ if mismatch is not None:
376+ return mismatch
377+ if self.auth is not None:
378+ mismatch = Contains("Authorization").match(request.headers)
379+ if mismatch is not None:
380+ return mismatch
381+ auth_value = request.headers["Authorization"]
382+ auth_scheme, auth_params = self.auth
383+ mismatch = StartsWith(auth_scheme + " ").match(auth_value)
384+ if mismatch is not None:
385+ return mismatch
386+ mismatch = Equals(auth_params).match(
387+ parse_dict_header(auth_value[len(auth_scheme + " "):]))
388+ if mismatch is not None:
389+ return mismatch
390+ if self.json_data is not None:
391+ mismatch = Equals(self.json_data).match(json.loads(request.body))
392+ if mismatch is not None:
393+ return mismatch
394+ if self.form_data is not None:
395+ if hasattr(request.body, "read"):
396+ body = request.body.read()
397+ else:
398+ body = request.body
399+ fs = FieldStorage(
400+ fp=io.BytesIO(body),
401+ environ={"REQUEST_METHOD": request.method},
402+ headers=request.headers)
403+ mismatch = MatchesDict(self.form_data).match(fs)
404+ if mismatch is not None:
405+ return mismatch
406+
407+
408+class TestSnapStoreClient(TestCaseWithFactory):
409+
410+ layer = LaunchpadZopelessLayer
411+
412+ def setUp(self):
413+ super(TestSnapStoreClient, self).setUp()
414+ self.useFixture(FeatureFixture(SNAP_TESTING_FLAGS))
415+ self.pushConfig(
416+ "snappy", store_url="http://sca.example/",
417+ store_upload_url="http://updown.example/")
418+ self.client = getUtility(ISnapStoreClient)
419+
420+ def test_requestPackageUploadPermission(self):
421+ @all_requests
422+ def handler(url, request):
423+ self.request = request
424+ return {"status_code": 200, "content": {"macaroon": "dummy"}}
425+
426+ snappy_series = self.factory.makeSnappySeries(name="rolling")
427+ with HTTMock(handler):
428+ macaroon = self.client.requestPackageUploadPermission(
429+ snappy_series, "test-snap")
430+ self.assertThat(self.request, RequestMatches(
431+ url=Equals("http://sca.example/dev/api/acl/"),
432+ method=Equals("POST"),
433+ json_data={
434+ "packages": [{"name": "test-snap", "series": "rolling"}],
435+ "permissions": ["package_upload"],
436+ }))
437+ self.assertEqual("dummy", macaroon)
438+ request = get_current_browser_request()
439+ start, stop = get_request_timeline(request).actions[-2:]
440+ self.assertEqual("request-snap-upload-macaroon-start", start.category)
441+ self.assertEqual("rolling/test-snap", start.detail)
442+ self.assertEqual("request-snap-upload-macaroon-stop", stop.category)
443+ self.assertEqual("rolling/test-snap", stop.detail)
444+
445+ def test_requestPackageUploadPermission_missing_macaroon(self):
446+ @all_requests
447+ def handler(url, request):
448+ return {"status_code": 200, "content": {}}
449+
450+ snappy_series = self.factory.makeSnappySeries()
451+ with HTTMock(handler):
452+ self.assertRaisesWithContent(
453+ BadRequestPackageUploadResponse, b"{}",
454+ self.client.requestPackageUploadPermission,
455+ snappy_series, "test-snap")
456+
457+ def test_requestPackageUploadPermission_404(self):
458+ @all_requests
459+ def handler(url, request):
460+ return {"status_code": 404, "reason": b"Not found"}
461+
462+ snappy_series = self.factory.makeSnappySeries()
463+ with HTTMock(handler):
464+ self.assertRaisesWithContent(
465+ BadRequestPackageUploadResponse,
466+ b"404 Client Error: Not found",
467+ self.client.requestPackageUploadPermission,
468+ snappy_series, "test-snap")
469+
470+ def test_upload(self):
471+ @urlmatch(path=r".*/unscanned-upload/$")
472+ def unscanned_upload_handler(url, request):
473+ self.unscanned_upload_request = request
474+ return {
475+ "status_code": 200,
476+ "content": {"successful": True, "upload_id": 1},
477+ }
478+
479+ @urlmatch(path=r".*/snap-upload/$")
480+ def snap_upload_handler(url, request):
481+ self.snap_upload_request = request
482+ return {"status_code": 202, "content": {"success": True}}
483+
484+ store_secrets = {"root": "dummy-root", "discharge": "dummy-discharge"}
485+ snap = self.factory.makeSnap(
486+ store_upload=True,
487+ store_series=self.factory.makeSnappySeries(name="rolling"),
488+ store_name="test-snap", store_secrets=store_secrets)
489+ snapbuild = self.factory.makeSnapBuild(snap=snap)
490+ lfa = self.factory.makeLibraryFileAlias(content="dummy snap content")
491+ self.factory.makeSnapFile(snapbuild=snapbuild, libraryfile=lfa)
492+ transaction.commit()
493+ with HTTMock(unscanned_upload_handler, snap_upload_handler):
494+ self.client.upload(snapbuild)
495+ self.assertThat(self.unscanned_upload_request, RequestMatches(
496+ url=Equals("http://updown.example/unscanned-upload/"),
497+ method=Equals("POST"),
498+ form_data={
499+ "binary": MatchesStructure.byEquality(
500+ name="binary", filename="filename",
501+ value="dummy snap content",
502+ type="application/octet-stream",
503+ )}))
504+ self.assertThat(self.snap_upload_request, RequestMatches(
505+ url=Equals("http://sca.example/dev/api/snap-upload/"),
506+ method=Equals("POST"), auth=("Macaroon", store_secrets),
507+ form_data={
508+ "name": MatchesStructure.byEquality(
509+ name="name", value="test-snap"),
510+ "updown_id": MatchesStructure.byEquality(
511+ name="updown_id", value="1"),
512+ "series": MatchesStructure.byEquality(
513+ name="series", value="rolling")}))
514
515=== modified file 'setup.py'
516--- setup.py 2016-03-16 02:08:40 +0000
517+++ setup.py 2016-05-12 10:54:06 +0000
518@@ -77,13 +77,14 @@
519 'paramiko',
520 'pgbouncer',
521 'psycopg2',
522- 'python-memcached',
523 'pyasn1',
524 'pystache',
525+ 'python-memcached',
526 'python-openid',
527 'pytz',
528 'rabbitfixture',
529 'requests',
530+ 'requests-toolbelt',
531 's4',
532 'setproctitle',
533 'setuptools',
534
535=== modified file 'versions.cfg'
536--- versions.cfg 2016-03-16 02:08:40 +0000
537+++ versions.cfg 2016-05-12 10:54:06 +0000
538@@ -119,6 +119,7 @@
539 PyYAML = 3.10
540 rabbitfixture = 0.3.6
541 requests = 2.7.0
542+requests-toolbelt = 0.6.2
543 s4 = 0.1.2
544 setproctitle = 1.1.7
545 setuptools-git = 1.0