Merge lp:~blake-rouse/maas/bootresource-simplestreams-endpoint into lp:~maas-committers/maas/trunk
- bootresource-simplestreams-endpoint
- Merge into trunk
Status: | Merged |
---|---|
Approved by: | Blake Rouse |
Approved revision: | no longer in the source branch. |
Merged at revision: | 2705 |
Proposed branch: | lp:~blake-rouse/maas/bootresource-simplestreams-endpoint |
Merge into: | lp:~maas-committers/maas/trunk |
Diff against target: |
745 lines (+700/-0) 4 files modified
src/maasserver/bootresources.py (+265/-0) src/maasserver/middleware.py (+3/-0) src/maasserver/tests/test_bootresources.py (+421/-0) src/maasserver/urls.py (+11/-0) |
To merge this branch: | bzr merge lp:~blake-rouse/maas/bootresource-simplestreams-endpoint |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Gavin Panella (community) | Approve | ||
Review via email: mp+230375@code.launchpad.net |
Commit message
Added simplestreams endpoint on the region controller at path images-stream. Cluster controllers will connect to this endpoint to download the boot resource image data.
Description of the change
Blake Rouse (blake-rouse) wrote : | # |
I will add some comments to the tests, fix the TransactionWrapper, and try to use django url routing.
Thanks for the review.
Gavin Panella (allenap) wrote : | # |
Tip top, thanks for the replies. I'm around tomorrow so ping me whenever you need a follow-up.
Blake Rouse (blake-rouse) wrote : | # |
Having an issue with the TransactionWrapper implementation you suggested. See inline comment.
Gavin Panella (allenap) : | # |
Gavin Panella (allenap) wrote : | # |
Changes look good.
MAAS Lander (maas-lander) wrote : | # |
The attempt to merge lp:~blake-rouse/maas/bootresource-simplestreams-endpoint into lp:maas failed. Below is the output from the failed tests.
Ign http://
Ign http://
Get:1 http://
Ign http://
Get:2 http://
Hit http://
Get:3 http://
Hit http://
Get:4 http://
Hit http://
Hit http://
Hit http://
Hit http://
Hit http://
Hit http://
Get:5 http://
Get:6 http://
Get:7 http://
Get:8 http://
Ign http://
Ign http://
Get:9 http://
Hit http://
Hit http://
Get:10 http://
Get:11 http://
Get:12 http://
Hit http://
Hit http://
Fetched 1,005 kB in 0s (1,920 kB/s)
Reading package lists...
sudo DEBIAN_
--
Preview Diff
1 | === added file 'src/maasserver/bootresources.py' | |||
2 | --- src/maasserver/bootresources.py 1970-01-01 00:00:00 +0000 | |||
3 | +++ src/maasserver/bootresources.py 2014-08-15 01:28:48 +0000 | |||
4 | @@ -0,0 +1,265 @@ | |||
5 | 1 | # Copyright 2014 Canonical Ltd. This software is licensed under the | ||
6 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | ||
7 | 3 | |||
8 | 4 | """Boot Resources.""" | ||
9 | 5 | |||
10 | 6 | from __future__ import ( | ||
11 | 7 | absolute_import, | ||
12 | 8 | print_function, | ||
13 | 9 | unicode_literals, | ||
14 | 10 | ) | ||
15 | 11 | |||
16 | 12 | str = None | ||
17 | 13 | |||
18 | 14 | __metaclass__ = type | ||
19 | 15 | __all__ = [ | ||
20 | 16 | "get_simplestream_endpoint", | ||
21 | 17 | "simplestreams_file_handler", | ||
22 | 18 | "simplestreams_stream_handler", | ||
23 | 19 | "SIMPLESTREAMS_URL_REGEXP", | ||
24 | 20 | ] | ||
25 | 21 | |||
26 | 22 | from django.db import ( | ||
27 | 23 | close_old_connections, | ||
28 | 24 | transaction, | ||
29 | 25 | ) | ||
30 | 26 | from django.http import ( | ||
31 | 27 | Http404, | ||
32 | 28 | HttpResponse, | ||
33 | 29 | StreamingHttpResponse, | ||
34 | 30 | ) | ||
35 | 31 | from django.shortcuts import get_object_or_404 | ||
36 | 32 | from maasserver.enum import BOOT_RESOURCE_TYPE | ||
37 | 33 | from maasserver.models import ( | ||
38 | 34 | BootResource, | ||
39 | 35 | BootResourceFile, | ||
40 | 36 | BootResourceSet, | ||
41 | 37 | ) | ||
42 | 38 | from maasserver.utils import absolute_reverse | ||
43 | 39 | from simplestreams import util as sutil | ||
44 | 40 | |||
45 | 41 | # Used by maasserver.middleware.AccessMiddleware to allow | ||
46 | 42 | # anonymous access to the simplestreams endpoint. | ||
47 | 43 | SIMPLESTREAMS_URL_REGEXP = '^/images-stream/' | ||
48 | 44 | |||
49 | 45 | |||
50 | 46 | def get_simplestream_endpoint(): | ||
51 | 47 | """Returns the simplestreams endpoint for the Region.""" | ||
52 | 48 | return { | ||
53 | 49 | 'url': absolute_reverse( | ||
54 | 50 | 'simplestreams_stream_handler', kwargs={'filename': 'index.json'}), | ||
55 | 51 | 'selections': [], | ||
56 | 52 | } | ||
57 | 53 | |||
58 | 54 | |||
59 | 55 | class TransactionWrapper: | ||
60 | 56 | """Wraps `LargeObjectFile` in transaction, so `StreamingHttpResponse` | ||
61 | 57 | can be used. Once the stream is done, then the transaction is | ||
62 | 58 | closed. | ||
63 | 59 | """ | ||
64 | 60 | |||
65 | 61 | def __init__(self, largeobject): | ||
66 | 62 | self.largeobject = largeobject | ||
67 | 63 | self._atomic = None | ||
68 | 64 | self._stream = None | ||
69 | 65 | |||
70 | 66 | def __iter__(self): | ||
71 | 67 | return self | ||
72 | 68 | |||
73 | 69 | def next(self): | ||
74 | 70 | if self._atomic is None: | ||
75 | 71 | self._atomic = transaction.atomic() | ||
76 | 72 | self._atomic.__enter__() | ||
77 | 73 | if self._stream is None: | ||
78 | 74 | self._stream = self.largeobject.open('rb') | ||
79 | 75 | data = self._stream.read(self.largeobject.block_size) | ||
80 | 76 | if len(data) == 0: | ||
81 | 77 | raise StopIteration | ||
82 | 78 | return data | ||
83 | 79 | |||
84 | 80 | def close(self): | ||
85 | 81 | if self._stream is not None: | ||
86 | 82 | self._stream.close() | ||
87 | 83 | self._stream = None | ||
88 | 84 | if self._atomic is not None: | ||
89 | 85 | self._atomic.__exit__() | ||
90 | 86 | self._atomic = None | ||
91 | 87 | close_old_connections() | ||
92 | 88 | |||
93 | 89 | |||
94 | 90 | class SimpleStreamsHandler: | ||
95 | 91 | """Simplestreams endpoint, that the clusters talk to. | ||
96 | 92 | |||
97 | 93 | This is not called from piston, as piston uses emitters which | ||
98 | 94 | breaks the ability to return streaming content. | ||
99 | 95 | |||
100 | 96 | Anyone can access this endpoint. No credentials are required. | ||
101 | 97 | """ | ||
102 | 98 | |||
103 | 99 | def get_json_response(self, content): | ||
104 | 100 | """Return `HttpResponse` for JSON content.""" | ||
105 | 101 | response = HttpResponse(content.encode('utf-8')) | ||
106 | 102 | response['Content-Type'] = "application/json" | ||
107 | 103 | return response | ||
108 | 104 | |||
109 | 105 | def get_boot_resource_identifiers(self, resource): | ||
110 | 106 | """Return tuple (os, arch, subarch, series) for the given resource.""" | ||
111 | 107 | arch, subarch = resource.split_arch() | ||
112 | 108 | if resource.rtype == BOOT_RESOURCE_TYPE.UPLOADED: | ||
113 | 109 | os = 'custom' | ||
114 | 110 | series = resource.name | ||
115 | 111 | else: | ||
116 | 112 | os, series = resource.name.split('/') | ||
117 | 113 | return (os, arch, subarch, series) | ||
118 | 114 | |||
119 | 115 | def get_product_name(self, resource): | ||
120 | 116 | """Return product name for the given resource.""" | ||
121 | 117 | return 'maas:boot:%s:%s:%s:%s' % self.get_boot_resource_identifiers( | ||
122 | 118 | resource) | ||
123 | 119 | |||
124 | 120 | def gen_complete_boot_resources(self): | ||
125 | 121 | """Return generator of `BootResource` that contains a complete set.""" | ||
126 | 122 | resources = BootResource.objects.all() | ||
127 | 123 | for resource in resources: | ||
128 | 124 | # Only add resources that have a complete set. | ||
129 | 125 | if resource.get_latest_complete_set() is None: | ||
130 | 126 | continue | ||
131 | 127 | yield resource | ||
132 | 128 | |||
133 | 129 | def gen_products_names(self): | ||
134 | 130 | """Return generator of avaliable products on the endpoint.""" | ||
135 | 131 | for resource in self.gen_complete_boot_resources(): | ||
136 | 132 | yield self.get_product_name(resource) | ||
137 | 133 | |||
138 | 134 | def get_product_index(self): | ||
139 | 135 | """Returns the streams product index `index.json`.""" | ||
140 | 136 | products = list(self.gen_products_names()) | ||
141 | 137 | updated = sutil.timestamp() | ||
142 | 138 | index = { | ||
143 | 139 | 'index': { | ||
144 | 140 | 'maas:v2:download': { | ||
145 | 141 | 'datatype': "image-downloads", | ||
146 | 142 | 'path': "streams/v1/maas:v2:download.json", | ||
147 | 143 | 'updated': updated, | ||
148 | 144 | 'products': products, | ||
149 | 145 | 'format': "products:1.0", | ||
150 | 146 | }, | ||
151 | 147 | }, | ||
152 | 148 | 'updated': updated, | ||
153 | 149 | 'format': "index:1.0" | ||
154 | 150 | } | ||
155 | 151 | data = sutil.dump_data(index) + "\n" | ||
156 | 152 | return self.get_json_response(data) | ||
157 | 153 | |||
158 | 154 | def get_product_item(self, resource, resource_set, rfile): | ||
159 | 155 | """Returns the item description for the `rfile`.""" | ||
160 | 156 | os, arch, subarch, series = self.get_boot_resource_identifiers( | ||
161 | 157 | resource) | ||
162 | 158 | path = '%s/%s/%s/%s/%s/%s' % ( | ||
163 | 159 | os, arch, subarch, series, resource_set.version, rfile.filename) | ||
164 | 160 | item = { | ||
165 | 161 | 'path': path, | ||
166 | 162 | 'ftype': rfile.filetype, | ||
167 | 163 | 'sha256': rfile.largefile.sha256, | ||
168 | 164 | 'size': rfile.largefile.total_size, | ||
169 | 165 | } | ||
170 | 166 | item.update(rfile.extra) | ||
171 | 167 | return item | ||
172 | 168 | |||
173 | 169 | def get_product_data(self, resource): | ||
174 | 170 | """Returns the product data for this resource.""" | ||
175 | 171 | os, arch, subarch, series = self.get_boot_resource_identifiers( | ||
176 | 172 | resource) | ||
177 | 173 | versions = {} | ||
178 | 174 | label = None | ||
179 | 175 | for resource_set in resource.sets.order_by('id').reverse(): | ||
180 | 176 | if not resource_set.complete: | ||
181 | 177 | continue | ||
182 | 178 | # Set the label to the latest complete set label. In most cases the | ||
183 | 179 | # label will be the same for all sets. Only time it will differ is | ||
184 | 180 | # when daily has been enabled for a resource, that was previously | ||
185 | 181 | # only release. Only the latest version of the resource will be | ||
186 | 182 | # downloaded. | ||
187 | 183 | if label is None: | ||
188 | 184 | label = resource_set.label | ||
189 | 185 | items = { | ||
190 | 186 | rfile.filename: self.get_product_item( | ||
191 | 187 | resource, resource_set, rfile) | ||
192 | 188 | for rfile in resource_set.files.all() | ||
193 | 189 | } | ||
194 | 190 | versions[resource_set.version] = { | ||
195 | 191 | 'items': items | ||
196 | 192 | } | ||
197 | 193 | product = { | ||
198 | 194 | 'versions': versions, | ||
199 | 195 | 'subarch': subarch, | ||
200 | 196 | 'label': label, | ||
201 | 197 | 'version': series, | ||
202 | 198 | 'arch': arch, | ||
203 | 199 | 'release': series, | ||
204 | 200 | 'krel': series, | ||
205 | 201 | 'os': os, | ||
206 | 202 | } | ||
207 | 203 | product.update(resource.extra) | ||
208 | 204 | return product | ||
209 | 205 | |||
210 | 206 | def get_product_download(self): | ||
211 | 207 | """Returns the streams download index `download.json`.""" | ||
212 | 208 | products = {} | ||
213 | 209 | for resource in self.gen_complete_boot_resources(): | ||
214 | 210 | name = self.get_product_name(resource) | ||
215 | 211 | products[name] = self.get_product_data(resource) | ||
216 | 212 | updated = sutil.timestamp() | ||
217 | 213 | index = { | ||
218 | 214 | 'datatype': "image-downloads", | ||
219 | 215 | 'updated': updated, | ||
220 | 216 | 'content_id': "maas:v2:download", | ||
221 | 217 | 'products': products, | ||
222 | 218 | 'format': "products:1.0" | ||
223 | 219 | } | ||
224 | 220 | data = sutil.dump_data(index) + "\n" | ||
225 | 221 | return self.get_json_response(data) | ||
226 | 222 | |||
227 | 223 | def streams_handler(self, request, filename): | ||
228 | 224 | """Handles requests into the "streams/" content.""" | ||
229 | 225 | if filename == 'index.json': | ||
230 | 226 | return self.get_product_index() | ||
231 | 227 | elif filename == 'maas:v2:download.json': | ||
232 | 228 | return self.get_product_download() | ||
233 | 229 | raise Http404() | ||
234 | 230 | |||
235 | 231 | def files_handler( | ||
236 | 232 | self, request, os, arch, subarch, series, version, filename): | ||
237 | 233 | """Handles requests for getting the boot resource data.""" | ||
238 | 234 | if os == "custom": | ||
239 | 235 | name = series | ||
240 | 236 | else: | ||
241 | 237 | name = '%s/%s' % (os, series) | ||
242 | 238 | arch = '%s/%s' % (arch, subarch) | ||
243 | 239 | resource = get_object_or_404( | ||
244 | 240 | BootResource, name=name, architecture=arch) | ||
245 | 241 | try: | ||
246 | 242 | resource_set = resource.sets.get(version=version) | ||
247 | 243 | except BootResourceSet.DoesNotExist: | ||
248 | 244 | raise Http404() | ||
249 | 245 | try: | ||
250 | 246 | rfile = resource_set.files.get(filename=filename) | ||
251 | 247 | except BootResourceFile.DoesNotExist: | ||
252 | 248 | raise Http404() | ||
253 | 249 | response = StreamingHttpResponse( | ||
254 | 250 | TransactionWrapper(rfile.largefile.content), | ||
255 | 251 | content_type='application/octet-stream') | ||
256 | 252 | response['Content-Length'] = rfile.largefile.total_size | ||
257 | 253 | return response | ||
258 | 254 | |||
259 | 255 | |||
260 | 256 | def simplestreams_stream_handler(request, filename): | ||
261 | 257 | handler = SimpleStreamsHandler() | ||
262 | 258 | return handler.streams_handler(request, filename) | ||
263 | 259 | |||
264 | 260 | |||
265 | 261 | def simplestreams_file_handler( | ||
266 | 262 | request, os, arch, subarch, series, version, filename): | ||
267 | 263 | handler = SimpleStreamsHandler() | ||
268 | 264 | return handler.files_handler( | ||
269 | 265 | request, os, arch, subarch, series, version, filename) | ||
270 | 0 | 266 | ||
271 | === modified file 'src/maasserver/middleware.py' | |||
272 | --- src/maasserver/middleware.py 2014-02-11 12:19:25 +0000 | |||
273 | +++ src/maasserver/middleware.py 2014-08-15 01:28:48 +0000 | |||
274 | @@ -52,6 +52,7 @@ | |||
275 | 52 | build_request_repr = repr | 52 | build_request_repr = repr |
276 | 53 | from django.utils.http import urlquote_plus | 53 | from django.utils.http import urlquote_plus |
277 | 54 | from maasserver import logger | 54 | from maasserver import logger |
278 | 55 | from maasserver.bootresources import SIMPLESTREAMS_URL_REGEXP | ||
279 | 55 | from maasserver.exceptions import ( | 56 | from maasserver.exceptions import ( |
280 | 56 | ExternalComponentException, | 57 | ExternalComponentException, |
281 | 57 | MAASAPIException, | 58 | MAASAPIException, |
282 | @@ -97,6 +98,8 @@ | |||
283 | 97 | reverse('metadata'), | 98 | reverse('metadata'), |
284 | 98 | # RPC information is for use by clusters; no login. | 99 | # RPC information is for use by clusters; no login. |
285 | 99 | reverse('rpc-info'), | 100 | reverse('rpc-info'), |
286 | 101 | # Boot resources simple streams endpoint; no login. | ||
287 | 102 | SIMPLESTREAMS_URL_REGEXP, | ||
288 | 100 | # API calls are protected by piston. | 103 | # API calls are protected by piston. |
289 | 101 | settings.API_URL_REGEXP, | 104 | settings.API_URL_REGEXP, |
290 | 102 | ] | 105 | ] |
291 | 103 | 106 | ||
292 | === added file 'src/maasserver/tests/test_bootresources.py' | |||
293 | --- src/maasserver/tests/test_bootresources.py 1970-01-01 00:00:00 +0000 | |||
294 | +++ src/maasserver/tests/test_bootresources.py 2014-08-15 01:28:48 +0000 | |||
295 | @@ -0,0 +1,421 @@ | |||
296 | 1 | # Copyright 2014 Canonical Ltd. This software is licensed under the | ||
297 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | ||
298 | 3 | |||
299 | 4 | """Test maasserver.bootresources.""" | ||
300 | 5 | |||
301 | 6 | from __future__ import ( | ||
302 | 7 | absolute_import, | ||
303 | 8 | print_function, | ||
304 | 9 | unicode_literals, | ||
305 | 10 | ) | ||
306 | 11 | |||
307 | 12 | str = None | ||
308 | 13 | |||
309 | 14 | __metaclass__ = type | ||
310 | 15 | __all__ = [] | ||
311 | 16 | |||
312 | 17 | import httplib | ||
313 | 18 | import json | ||
314 | 19 | from random import randint | ||
315 | 20 | |||
316 | 21 | from django.core.urlresolvers import reverse | ||
317 | 22 | from django.db import transaction | ||
318 | 23 | from django.http import StreamingHttpResponse | ||
319 | 24 | from django.test.client import Client | ||
320 | 25 | from maasserver.bootresources import get_simplestream_endpoint | ||
321 | 26 | from maasserver.enum import ( | ||
322 | 27 | BOOT_RESOURCE_FILE_TYPE, | ||
323 | 28 | BOOT_RESOURCE_TYPE, | ||
324 | 29 | ) | ||
325 | 30 | from maasserver.testing.factory import factory | ||
326 | 31 | from maasserver.testing.testcase import MAASServerTestCase | ||
327 | 32 | from maasserver.utils import absolute_reverse | ||
328 | 33 | from maastesting.testcase import MAASTestCase | ||
329 | 34 | from testtools.matchers import ContainsAll | ||
330 | 35 | |||
331 | 36 | |||
332 | 37 | class TestHelpers(MAASServerTestCase): | ||
333 | 38 | """Tests for `maasserver.bootresources` helpers.""" | ||
334 | 39 | |||
335 | 40 | def test_get_simplestreams_endpoint(self): | ||
336 | 41 | endpoint = get_simplestream_endpoint() | ||
337 | 42 | self.assertEqual( | ||
338 | 43 | absolute_reverse( | ||
339 | 44 | 'simplestreams_stream_handler', | ||
340 | 45 | kwargs={'filename': 'index.json'}), | ||
341 | 46 | endpoint['url']) | ||
342 | 47 | self.assertEqual([], endpoint['selections']) | ||
343 | 48 | |||
344 | 49 | |||
345 | 50 | class TestSimpleStreamsHandler(MAASServerTestCase): | ||
346 | 51 | """Tests for `maasserver.bootresources.SimpleStreamsHandler`.""" | ||
347 | 52 | |||
348 | 53 | def reverse_stream_handler(self, filename): | ||
349 | 54 | return reverse( | ||
350 | 55 | 'simplestreams_stream_handler', kwargs={'filename': filename}) | ||
351 | 56 | |||
352 | 57 | def reverse_file_handler( | ||
353 | 58 | self, os, arch, subarch, series, version, filename): | ||
354 | 59 | return reverse( | ||
355 | 60 | 'simplestreams_file_handler', kwargs={ | ||
356 | 61 | 'os': os, | ||
357 | 62 | 'arch': arch, | ||
358 | 63 | 'subarch': subarch, | ||
359 | 64 | 'series': series, | ||
360 | 65 | 'version': version, | ||
361 | 66 | 'filename': filename, | ||
362 | 67 | }) | ||
363 | 68 | |||
364 | 69 | def get_stream_client(self, filename): | ||
365 | 70 | return self.client.get(self.reverse_stream_handler(filename)) | ||
366 | 71 | |||
367 | 72 | def get_file_client(self, os, arch, subarch, series, version, filename): | ||
368 | 73 | return self.client.get( | ||
369 | 74 | self.reverse_file_handler( | ||
370 | 75 | os, arch, subarch, series, version, filename)) | ||
371 | 76 | |||
372 | 77 | def get_product_name_for_resource(self, resource): | ||
373 | 78 | arch, subarch = resource.architecture.split('/') | ||
374 | 79 | if resource.rtype == BOOT_RESOURCE_TYPE.UPLOADED: | ||
375 | 80 | os = 'custom' | ||
376 | 81 | series = resource.name | ||
377 | 82 | else: | ||
378 | 83 | os, series = resource.name.split('/') | ||
379 | 84 | return 'maas:boot:%s:%s:%s:%s' % (os, arch, subarch, series) | ||
380 | 85 | |||
381 | 86 | def make_usable_product_boot_resource(self): | ||
382 | 87 | resource = factory.make_usable_boot_resource() | ||
383 | 88 | return self.get_product_name_for_resource(resource), resource | ||
384 | 89 | |||
385 | 90 | def test_streams_other_than_allowed_returns_404(self): | ||
386 | 91 | allowed_paths = [ | ||
387 | 92 | 'index.json', | ||
388 | 93 | 'maas:v2:download.json', | ||
389 | 94 | ] | ||
390 | 95 | invalid_paths = [ | ||
391 | 96 | '%s.json' % factory.make_name('path') | ||
392 | 97 | for _ in range(3) | ||
393 | 98 | ] | ||
394 | 99 | for path in allowed_paths: | ||
395 | 100 | response = self.get_stream_client(path) | ||
396 | 101 | self.assertEqual(httplib.OK, response.status_code) | ||
397 | 102 | for path in invalid_paths: | ||
398 | 103 | response = self.get_stream_client(path) | ||
399 | 104 | self.assertEqual(httplib.NOT_FOUND, response.status_code) | ||
400 | 105 | |||
401 | 106 | def test_streams_product_index_contains_keys(self): | ||
402 | 107 | response = self.get_stream_client('index.json') | ||
403 | 108 | output = json.loads(response.content) | ||
404 | 109 | self.assertThat(output, ContainsAll(['index', 'updated', 'format'])) | ||
405 | 110 | |||
406 | 111 | def test_streams_product_index_format_is_index_1(self): | ||
407 | 112 | response = self.get_stream_client('index.json') | ||
408 | 113 | output = json.loads(response.content) | ||
409 | 114 | self.assertEqual('index:1.0', output['format']) | ||
410 | 115 | |||
411 | 116 | def test_streams_product_index_index_has_maas_v2_download(self): | ||
412 | 117 | response = self.get_stream_client('index.json') | ||
413 | 118 | output = json.loads(response.content) | ||
414 | 119 | self.assertThat(output['index'], ContainsAll(['maas:v2:download'])) | ||
415 | 120 | |||
416 | 121 | def test_streams_product_index_maas_v2_download_contains_keys(self): | ||
417 | 122 | response = self.get_stream_client('index.json') | ||
418 | 123 | output = json.loads(response.content) | ||
419 | 124 | self.assertThat( | ||
420 | 125 | output['index']['maas:v2:download'], | ||
421 | 126 | ContainsAll([ | ||
422 | 127 | 'datatype', 'path', 'updated', 'products', 'format'])) | ||
423 | 128 | |||
424 | 129 | def test_streams_product_index_maas_v2_download_has_valid_values(self): | ||
425 | 130 | response = self.get_stream_client('index.json') | ||
426 | 131 | output = json.loads(response.content) | ||
427 | 132 | self.assertEqual( | ||
428 | 133 | 'image-downloads', | ||
429 | 134 | output['index']['maas:v2:download']['datatype']) | ||
430 | 135 | self.assertEqual( | ||
431 | 136 | 'streams/v1/maas:v2:download.json', | ||
432 | 137 | output['index']['maas:v2:download']['path']) | ||
433 | 138 | self.assertEqual( | ||
434 | 139 | 'products:1.0', | ||
435 | 140 | output['index']['maas:v2:download']['format']) | ||
436 | 141 | |||
437 | 142 | def test_streams_product_index_empty_products(self): | ||
438 | 143 | response = self.get_stream_client('index.json') | ||
439 | 144 | output = json.loads(response.content) | ||
440 | 145 | self.assertEqual( | ||
441 | 146 | [], | ||
442 | 147 | output['index']['maas:v2:download']['products']) | ||
443 | 148 | |||
444 | 149 | def test_streams_product_index_empty_with_incomplete_resource(self): | ||
445 | 150 | resource = factory.make_boot_resource() | ||
446 | 151 | factory.make_boot_resource_set(resource) | ||
447 | 152 | response = self.get_stream_client('index.json') | ||
448 | 153 | output = json.loads(response.content) | ||
449 | 154 | self.assertEqual( | ||
450 | 155 | [], | ||
451 | 156 | output['index']['maas:v2:download']['products']) | ||
452 | 157 | |||
453 | 158 | def test_streams_product_index_with_resources(self): | ||
454 | 159 | products = [] | ||
455 | 160 | for _ in range(3): | ||
456 | 161 | product, _ = self.make_usable_product_boot_resource() | ||
457 | 162 | products.append(product) | ||
458 | 163 | response = self.get_stream_client('index.json') | ||
459 | 164 | output = json.loads(response.content) | ||
460 | 165 | # Product listing should be the same as all of the completed | ||
461 | 166 | # boot resources in the database. | ||
462 | 167 | self.assertItemsEqual( | ||
463 | 168 | products, | ||
464 | 169 | output['index']['maas:v2:download']['products']) | ||
465 | 170 | |||
466 | 171 | def test_streams_product_download_contains_keys(self): | ||
467 | 172 | response = self.get_stream_client('maas:v2:download.json') | ||
468 | 173 | output = json.loads(response.content) | ||
469 | 174 | self.assertThat(output, ContainsAll([ | ||
470 | 175 | 'datatype', 'updated', 'content_id', 'products', 'format'])) | ||
471 | 176 | |||
472 | 177 | def test_streams_product_download_has_valid_values(self): | ||
473 | 178 | response = self.get_stream_client('maas:v2:download.json') | ||
474 | 179 | output = json.loads(response.content) | ||
475 | 180 | self.assertEqual('image-downloads', output['datatype']) | ||
476 | 181 | self.assertEqual('maas:v2:download', output['content_id']) | ||
477 | 182 | self.assertEqual('products:1.0', output['format']) | ||
478 | 183 | |||
479 | 184 | def test_streams_product_download_empty_products(self): | ||
480 | 185 | response = self.get_stream_client('maas:v2:download.json') | ||
481 | 186 | output = json.loads(response.content) | ||
482 | 187 | self.assertEqual( | ||
483 | 188 | {}, | ||
484 | 189 | output['products']) | ||
485 | 190 | |||
486 | 191 | def test_streams_product_download_empty_with_incomplete_resource(self): | ||
487 | 192 | resource = factory.make_boot_resource() | ||
488 | 193 | factory.make_boot_resource_set(resource) | ||
489 | 194 | response = self.get_stream_client('maas:v2:download.json') | ||
490 | 195 | output = json.loads(response.content) | ||
491 | 196 | self.assertEqual( | ||
492 | 197 | {}, | ||
493 | 198 | output['products']) | ||
494 | 199 | |||
495 | 200 | def test_streams_product_download_has_valid_product_keys(self): | ||
496 | 201 | products = [] | ||
497 | 202 | for _ in range(3): | ||
498 | 203 | product, _ = self.make_usable_product_boot_resource() | ||
499 | 204 | products.append(product) | ||
500 | 205 | response = self.get_stream_client('maas:v2:download.json') | ||
501 | 206 | output = json.loads(response.content) | ||
502 | 207 | # Product listing should be the same as all of the completed | ||
503 | 208 | # boot resources in the database. | ||
504 | 209 | self.assertThat( | ||
505 | 210 | output['products'], | ||
506 | 211 | ContainsAll(products)) | ||
507 | 212 | |||
508 | 213 | def test_streams_product_download_product_contains_keys(self): | ||
509 | 214 | product, _ = self.make_usable_product_boot_resource() | ||
510 | 215 | response = self.get_stream_client('maas:v2:download.json') | ||
511 | 216 | output = json.loads(response.content) | ||
512 | 217 | self.assertThat( | ||
513 | 218 | output['products'][product], | ||
514 | 219 | ContainsAll([ | ||
515 | 220 | 'versions', 'subarch', 'label', 'version', | ||
516 | 221 | 'arch', 'release', 'krel', 'os'])) | ||
517 | 222 | |||
518 | 223 | def test_streams_product_download_product_has_valid_values(self): | ||
519 | 224 | product, resource = self.make_usable_product_boot_resource() | ||
520 | 225 | _, _, os, arch, subarch, series = product.split(':') | ||
521 | 226 | label = resource.get_latest_complete_set().label | ||
522 | 227 | response = self.get_stream_client('maas:v2:download.json') | ||
523 | 228 | output = json.loads(response.content) | ||
524 | 229 | output_product = output['products'][product] | ||
525 | 230 | self.assertEqual(subarch, output_product['subarch']) | ||
526 | 231 | self.assertEqual(label, output_product['label']) | ||
527 | 232 | self.assertEqual(series, output_product['version']) | ||
528 | 233 | self.assertEqual(arch, output_product['arch']) | ||
529 | 234 | self.assertEqual(series, output_product['release']) | ||
530 | 235 | self.assertEqual(series, output_product['krel']) | ||
531 | 236 | self.assertEqual(os, output_product['os']) | ||
532 | 237 | for key, value in resource.extra.items(): | ||
533 | 238 | self.assertEqual(value, output_product[key]) | ||
534 | 239 | |||
535 | 240 | def test_streams_product_download_product_uses_latest_complete_label(self): | ||
536 | 241 | product, resource = self.make_usable_product_boot_resource() | ||
537 | 242 | # Incomplete resource_set | ||
538 | 243 | factory.make_boot_resource_set(resource) | ||
539 | 244 | newest_set = factory.make_boot_resource_set(resource) | ||
540 | 245 | factory.make_boot_resource_file_with_content(newest_set) | ||
541 | 246 | response = self.get_stream_client('maas:v2:download.json') | ||
542 | 247 | output = json.loads(response.content) | ||
543 | 248 | output_product = output['products'][product] | ||
544 | 249 | self.assertEqual(newest_set.label, output_product['label']) | ||
545 | 250 | |||
546 | 251 | def test_streams_product_download_product_contains_multiple_versions(self): | ||
547 | 252 | resource = factory.make_boot_resource() | ||
548 | 253 | resource_sets = [ | ||
549 | 254 | factory.make_boot_resource_set(resource) | ||
550 | 255 | for _ in range(3) | ||
551 | 256 | ] | ||
552 | 257 | versions = [] | ||
553 | 258 | for resource_set in resource_sets: | ||
554 | 259 | factory.make_boot_resource_file_with_content(resource_set) | ||
555 | 260 | versions.append(resource_set.version) | ||
556 | 261 | product = self.get_product_name_for_resource(resource) | ||
557 | 262 | response = self.get_stream_client('maas:v2:download.json') | ||
558 | 263 | output = json.loads(response.content) | ||
559 | 264 | self.assertThat( | ||
560 | 265 | output['products'][product]['versions'], | ||
561 | 266 | ContainsAll(versions)) | ||
562 | 267 | |||
563 | 268 | def test_streams_product_download_product_version_contains_items(self): | ||
564 | 269 | product, resource = self.make_usable_product_boot_resource() | ||
565 | 270 | resource_set = resource.get_latest_complete_set() | ||
566 | 271 | items = [ | ||
567 | 272 | rfile.filename | ||
568 | 273 | for rfile in resource_set.files.all() | ||
569 | 274 | ] | ||
570 | 275 | response = self.get_stream_client('maas:v2:download.json') | ||
571 | 276 | output = json.loads(response.content) | ||
572 | 277 | version = output['products'][product]['versions'][resource_set.version] | ||
573 | 278 | self.assertThat( | ||
574 | 279 | version['items'], | ||
575 | 280 | ContainsAll(items)) | ||
576 | 281 | |||
577 | 282 | def test_streams_product_download_product_item_contains_keys(self): | ||
578 | 283 | product, resource = self.make_usable_product_boot_resource() | ||
579 | 284 | resource_set = resource.get_latest_complete_set() | ||
580 | 285 | resource_file = resource_set.files.order_by('?')[0] | ||
581 | 286 | response = self.get_stream_client('maas:v2:download.json') | ||
582 | 287 | output = json.loads(response.content) | ||
583 | 288 | version = output['products'][product]['versions'][resource_set.version] | ||
584 | 289 | self.assertThat( | ||
585 | 290 | version['items'][resource_file.filename], | ||
586 | 291 | ContainsAll(['path', 'ftype', 'sha256', 'size'])) | ||
587 | 292 | |||
588 | 293 | def test_streams_product_download_product_item_has_valid_values(self): | ||
589 | 294 | product, resource = self.make_usable_product_boot_resource() | ||
590 | 295 | _, _, os, arch, subarch, series = product.split(':') | ||
591 | 296 | resource_set = resource.get_latest_complete_set() | ||
592 | 297 | resource_file = resource_set.files.order_by('?')[0] | ||
593 | 298 | path = '%s/%s/%s/%s/%s/%s' % ( | ||
594 | 299 | os, arch, subarch, series, resource_set.version, | ||
595 | 300 | resource_file.filename) | ||
596 | 301 | response = self.get_stream_client('maas:v2:download.json') | ||
597 | 302 | output = json.loads(response.content) | ||
598 | 303 | version = output['products'][product]['versions'][resource_set.version] | ||
599 | 304 | item = version['items'][resource_file.filename] | ||
600 | 305 | self.assertEqual(path, item['path']) | ||
601 | 306 | self.assertEqual(resource_file.filetype, item['ftype']) | ||
602 | 307 | self.assertEqual(resource_file.largefile.sha256, item['sha256']) | ||
603 | 308 | self.assertEqual(resource_file.largefile.total_size, item['size']) | ||
604 | 309 | for key, value in resource_file.extra.items(): | ||
605 | 310 | self.assertEqual(value, item[key]) | ||
606 | 311 | |||
607 | 312 | def test_download_invalid_boot_resource_returns_404(self): | ||
608 | 313 | os = factory.make_name('os') | ||
609 | 314 | series = factory.make_name('series') | ||
610 | 315 | arch = factory.make_name('arch') | ||
611 | 316 | subarch = factory.make_name('subarch') | ||
612 | 317 | version = factory.make_name('version') | ||
613 | 318 | filename = factory.make_name('filename') | ||
614 | 319 | response = self.get_file_client( | ||
615 | 320 | os, arch, subarch, series, version, filename) | ||
616 | 321 | self.assertEqual(httplib.NOT_FOUND, response.status_code) | ||
617 | 322 | |||
618 | 323 | def test_download_invalid_version_returns_404(self): | ||
619 | 324 | product, resource = self.make_usable_product_boot_resource() | ||
620 | 325 | _, _, os, arch, subarch, series = product.split(':') | ||
621 | 326 | version = factory.make_name('version') | ||
622 | 327 | filename = factory.make_name('filename') | ||
623 | 328 | response = self.get_file_client( | ||
624 | 329 | os, arch, subarch, series, version, filename) | ||
625 | 330 | self.assertEqual(httplib.NOT_FOUND, response.status_code) | ||
626 | 331 | |||
627 | 332 | def test_download_invalid_filename_returns_404(self): | ||
628 | 333 | product, resource = self.make_usable_product_boot_resource() | ||
629 | 334 | _, _, os, arch, subarch, series = product.split(':') | ||
630 | 335 | resource_set = resource.get_latest_complete_set() | ||
631 | 336 | version = resource_set.version | ||
632 | 337 | filename = factory.make_name('filename') | ||
633 | 338 | response = self.get_file_client( | ||
634 | 339 | os, arch, subarch, series, version, filename) | ||
635 | 340 | self.assertEqual(httplib.NOT_FOUND, response.status_code) | ||
636 | 341 | |||
637 | 342 | def test_download_valid_path_returns_200(self): | ||
638 | 343 | product, resource = self.make_usable_product_boot_resource() | ||
639 | 344 | _, _, os, arch, subarch, series = product.split(':') | ||
640 | 345 | resource_set = resource.get_latest_complete_set() | ||
641 | 346 | version = resource_set.version | ||
642 | 347 | resource_file = resource_set.files.order_by('?')[0] | ||
643 | 348 | filename = resource_file.filename | ||
644 | 349 | response = self.get_file_client( | ||
645 | 350 | os, arch, subarch, series, version, filename) | ||
646 | 351 | self.assertEqual(httplib.OK, response.status_code) | ||
647 | 352 | |||
648 | 353 | def test_download_returns_streaming_response(self): | ||
649 | 354 | product, resource = self.make_usable_product_boot_resource() | ||
650 | 355 | _, _, os, arch, subarch, series = product.split(':') | ||
651 | 356 | resource_set = resource.get_latest_complete_set() | ||
652 | 357 | version = resource_set.version | ||
653 | 358 | resource_file = resource_set.files.order_by('?')[0] | ||
654 | 359 | filename = resource_file.filename | ||
655 | 360 | with resource_file.largefile.content.open('rb') as stream: | ||
656 | 361 | content = stream.read() | ||
657 | 362 | response = self.get_file_client( | ||
658 | 363 | os, arch, subarch, series, version, filename) | ||
659 | 364 | self.assertIsInstance(response, StreamingHttpResponse) | ||
660 | 365 | self.assertEqual(content, b''.join(response.streaming_content)) | ||
661 | 366 | |||
662 | 367 | |||
663 | 368 | class TestTransactionWrapper(MAASTestCase): | ||
664 | 369 | """Tests the use of StreamingHttpResponse(TransactionWrapper(stream)). | ||
665 | 370 | |||
666 | 371 | We do not run this inside of `MAASServerTestCase` as that wraps a | ||
667 | 372 | transaction around each test. This removes that behavior so we can | ||
668 | 373 | test that the transaction is remaining open for all of the content. | ||
669 | 374 | """ | ||
670 | 375 | |||
671 | 376 | def test_download(self): | ||
672 | 377 | # Do the setup inside of a transaction, as we are running in a test | ||
673 | 378 | # that doesn't enable transactions per test. | ||
674 | 379 | with transaction.atomic(): | ||
675 | 380 | os = factory.make_name('os') | ||
676 | 381 | series = factory.make_name('series') | ||
677 | 382 | arch = factory.make_name('arch') | ||
678 | 383 | subarch = factory.make_name('subarch') | ||
679 | 384 | name = '%s/%s' % (os, series) | ||
680 | 385 | architecture = '%s/%s' % (arch, subarch) | ||
681 | 386 | version = factory.make_name('version') | ||
682 | 387 | filetype = factory.pick_enum(BOOT_RESOURCE_FILE_TYPE) | ||
683 | 388 | # We set the filename to the same value as filetype, as in most | ||
684 | 389 | # cases this will always be true. The simplestreams content from | ||
685 | 390 | # maas.ubuntu.com, is formatted this way. | ||
686 | 391 | filename = filetype | ||
687 | 392 | size = randint(1024, 2048) | ||
688 | 393 | content = factory.make_string(size=size) | ||
689 | 394 | resource = factory.make_boot_resource( | ||
690 | 395 | rtype=BOOT_RESOURCE_TYPE.SYNCED, name=name, | ||
691 | 396 | architecture=architecture) | ||
692 | 397 | resource_set = factory.make_boot_resource_set( | ||
693 | 398 | resource, version=version) | ||
694 | 399 | largefile = factory.make_large_file(content=content, size=size) | ||
695 | 400 | factory.make_boot_resource_file( | ||
696 | 401 | resource_set, largefile, filename=filename, filetype=filetype) | ||
697 | 402 | |||
698 | 403 | # Outside of the transaction, we run the actual test. The client will | ||
699 | 404 | # run inside of its own transaction, but once the streaming response | ||
700 | 405 | # is returned that transaction will be closed. | ||
701 | 406 | client = Client() | ||
702 | 407 | response = client.get( | ||
703 | 408 | reverse( | ||
704 | 409 | 'simplestreams_file_handler', kwargs={ | ||
705 | 410 | 'os': os, | ||
706 | 411 | 'arch': arch, | ||
707 | 412 | 'subarch': subarch, | ||
708 | 413 | 'series': series, | ||
709 | 414 | 'version': version, | ||
710 | 415 | 'filename': filename, | ||
711 | 416 | })) | ||
712 | 417 | |||
713 | 418 | # If TransactionWrapper does not work, then a ProgramError will be | ||
714 | 419 | # thrown. If it works then content will match. | ||
715 | 420 | self.assertEqual(content, b''.join(response.streaming_content)) | ||
716 | 421 | self.assertTrue(largefile.content.closed) | ||
717 | 0 | 422 | ||
718 | === modified file 'src/maasserver/urls.py' | |||
719 | --- src/maasserver/urls.py 2014-08-13 21:49:35 +0000 | |||
720 | +++ src/maasserver/urls.py 2014-08-15 01:28:48 +0000 | |||
721 | @@ -21,6 +21,10 @@ | |||
722 | 21 | url, | 21 | url, |
723 | 22 | ) | 22 | ) |
724 | 23 | from django.contrib.auth.decorators import user_passes_test | 23 | from django.contrib.auth.decorators import user_passes_test |
725 | 24 | from maasserver.bootresources import ( | ||
726 | 25 | simplestreams_file_handler, | ||
727 | 26 | simplestreams_stream_handler, | ||
728 | 27 | ) | ||
729 | 24 | from maasserver.enum import NODEGROUP_STATUS | 28 | from maasserver.enum import NODEGROUP_STATUS |
730 | 25 | from maasserver.models import Node | 29 | from maasserver.models import Node |
731 | 26 | from maasserver.views import TextTemplateView | 30 | from maasserver.views import TextTemplateView |
732 | @@ -104,6 +108,13 @@ | |||
733 | 104 | 'maasserver.views', | 108 | 'maasserver.views', |
734 | 105 | url(r'^accounts/login/$', login, name='login'), | 109 | url(r'^accounts/login/$', login, name='login'), |
735 | 106 | url( | 110 | url( |
736 | 111 | r'^images-stream/streams/v1/(?P<filename>.*)$', | ||
737 | 112 | simplestreams_stream_handler, name='simplestreams_stream_handler'), | ||
738 | 113 | url( | ||
739 | 114 | r'^images-stream/(?P<os>.*)/(?P<arch>.*)/(?P<subarch>.*)/' | ||
740 | 115 | '(?P<series>.*)/(?P<version>.*)/(?P<filename>.*)$', | ||
741 | 116 | simplestreams_file_handler, name='simplestreams_file_handler'), | ||
742 | 117 | url( | ||
743 | 107 | r'^robots\.txt$', TextTemplateView.as_view( | 118 | r'^robots\.txt$', TextTemplateView.as_view( |
744 | 108 | template_name='maasserver/robots.txt'), | 119 | template_name='maasserver/robots.txt'), |
745 | 109 | name='robots'), | 120 | name='robots'), |
Looks good... but I don't understand a lot of it.
The branch is a bit too big. I don't mean just line count, I mean that there's a lot going on. Things like TransactionWrapper and its tests could probably have been landed separately, and SimpleStreamsHa ndler could have been landed/reviewed in stages.
Pretty much all tests could do with some comments to explain what's going on. Remember that I'm you in about a month, when you've lost all the context you've currently got in-brain.
Basically it _looks_ fine, but add some comments, please :) Also, I have a suggestion for a simpler TransactionWrapper.