Merge lp:~ltrager/maas-images/bootloaders_part2 into lp:maas-images
- bootloaders_part2
- Merge into maas-ephemerals
Proposed by
Lee Trager
Status: | Merged |
---|---|
Merged at revision: | 325 |
Proposed branch: | lp:~ltrager/maas-images/bootloaders_part2 |
Merge into: | lp:maas-images |
Diff against target: |
485 lines (+231/-104) 3 files modified
conf/bootloaders.yaml (+21/-13) meph2/commands/dpkg.py (+133/-8) meph2/commands/meph2_util.py (+77/-83) |
To merge this branch: | bzr merge lp:~ltrager/maas-images/bootloaders_part2 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Scott Moser (community) | Approve | ||
Review via email:
|
Commit message
Description of the change
Update bootloader format as per discussion in https:/
Sample output:
http://
To post a comment you must log in.
- 324. By Lee Trager
-
Put os in bootloader product name
- 325. By Lee Trager
-
Use bootloaders from Xenial
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Lee Trager (ltrager) wrote : | # |
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Scott Moser (smoser) wrote : | # |
the data looks good.
i have one nit pick in the conf/bootloader
I think better to have:
files:
- usr/lib/
than
files:
- /usr/lib/
just to make it clear that the files are coming from the package itself and not the isntalled system, and also to avoid the pitfall of:
os.path.
review:
Approve
- 326. By Lee Trager
-
Remove leading / from bootloader files
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'conf/bootloaders.yaml' |
2 | --- conf/bootloaders.yaml 2016-08-23 15:17:16 +0000 |
3 | +++ conf/bootloaders.yaml 2016-08-24 20:40:59 +0000 |
4 | @@ -1,35 +1,41 @@ |
5 | -product_id: "com.ubuntu.maas.daily:1:bootloaders-bases:{bootloader}" |
6 | -content_id: "com.ubuntu.maas:daily:1:bootloaders-bases-download" |
7 | +product_id: "com.ubuntu.maas.daily:1:{os}:{firmware_platform}:{arch}" |
8 | +content_id: "com.ubuntu.maas:daily:1:bootloader-download" |
9 | |
10 | bootloaders: |
11 | - - bootloader: pxe |
12 | + - firmware-platform: pxe |
13 | packages: |
14 | - pxelinux |
15 | - syslinux-common |
16 | - arch: amd64 |
17 | + arch: i386 |
18 | + arches: i386,amd64 |
19 | archive: http://archive.ubuntu.com/ubuntu |
20 | release: xenial |
21 | + os: pxelinux |
22 | files: |
23 | - - /usr/lib/PXELINUX/pxelinux.0 |
24 | - - /usr/lib/syslinux/modules/bios/*.c32 |
25 | - - bootloader: uefi |
26 | + - usr/lib/PXELINUX/pxelinux.0 |
27 | + - usr/lib/syslinux/modules/bios/*.c32 |
28 | + - firmware-platform: uefi |
29 | packages: |
30 | - shim-signed |
31 | - grub-efi-amd64-signed |
32 | arch: amd64 |
33 | + arches: amd64 |
34 | archive: http://archive.ubuntu.com/ubuntu |
35 | release: xenial |
36 | + os: grub-efi-signed |
37 | files: |
38 | - - /usr/lib/shim/shim.efi.signed, bootx64.efi |
39 | - - /usr/lib/grub/x86_64-efi-signed/grubnetx64.efi.signed, grubx64.efi |
40 | - - bootloader: uefi-arm64 |
41 | + - usr/lib/shim/shim.efi.signed, bootx64.efi |
42 | + - usr/lib/grub/x86_64-efi-signed/grubnetx64.efi.signed, grubx64.efi |
43 | + - firmware-platform: uefi |
44 | packages: |
45 | - grub-efi-arm64-bin |
46 | arch: arm64 |
47 | + arches: arm64 |
48 | archive: http://ports.ubuntu.com |
49 | release: xenial |
50 | + os: grub-efi |
51 | files: |
52 | - - /usr/lib/grub/arm64-efi/ |
53 | + - usr/lib/grub/arm64-efi/ |
54 | grub_format: arm64-efi |
55 | grub_output: grubaa64.efi |
56 | grub_config: | |
57 | @@ -41,14 +47,16 @@ |
58 | # Failed to load based on MAC address. |
59 | # Load arm64 by default, UEFI only supported by 64-bit |
60 | configfile (pxe)/grub/grub.cfg-default-arm64 |
61 | - - bootloader: powerkvm |
62 | + - firmware-platform: open-firmware |
63 | packages: |
64 | - grub-ieee1275-bin |
65 | arch: ppc64el |
66 | + arches: ppc64el,ppc64 |
67 | archive: http://ports.ubuntu.com |
68 | release: xenial |
69 | + os: grub-ieee1275 |
70 | files: |
71 | - - /usr/lib/grub/powerpc-ieee1275/ |
72 | + - usr/lib/grub/powerpc-ieee1275/ |
73 | grub_format: powerpc-ieee1275 |
74 | grub_output: bootppc64.bin |
75 | grub_config: | |
76 | |
77 | === modified file 'meph2/commands/dpkg.py' |
78 | --- meph2/commands/dpkg.py 2016-08-03 13:00:03 +0000 |
79 | +++ meph2/commands/dpkg.py 2016-08-24 20:40:59 +0000 |
80 | @@ -6,6 +6,7 @@ |
81 | import os |
82 | import re |
83 | import sys |
84 | +import tarfile |
85 | import tempfile |
86 | import glob |
87 | |
88 | @@ -128,13 +129,98 @@ |
89 | pkg_path = os.path.join(dest, os.path.basename(package['Filename'])) |
90 | with open(pkg_path, 'wb') as stream: |
91 | stream.write(pkg_data) |
92 | + package['files'] = [] |
93 | + output = subprocess.check_output(['dpkg', '-c', pkg_path]) |
94 | + for line in output.decode('utf-8').split('\n'): |
95 | + # The file is the last column in the list. |
96 | + file_info = line.split() |
97 | + # Last line is just a newline |
98 | + if len(file_info) == 0: |
99 | + continue |
100 | + if file_info[-1].startswith('./'): |
101 | + # Remove leading './' if it exists |
102 | + f = file_info[-1][2:] |
103 | + elif file_info[-1].startswith('/'): |
104 | + # Removing leading '/' if it exists |
105 | + f = file_info[-1][1:] |
106 | + else: |
107 | + f = file_info[-1] |
108 | + if f != '': |
109 | + package['files'].append(f) |
110 | return package |
111 | |
112 | |
113 | +def get_file_info(f): |
114 | + size = 0 |
115 | + sha256 = hashlib.sha256() |
116 | + with open(f, 'rb') as f: |
117 | + for chunk in iter(lambda: f.read(2**15), b''): |
118 | + sha256.update(chunk) |
119 | + size += len(chunk) |
120 | + return sha256.hexdigest(), size |
121 | + |
122 | + |
123 | +def make_item(ftype, src_file, dest_file, stream_path, src_packages): |
124 | + sha256, size = get_file_info(dest_file) |
125 | + for src_package in src_packages: |
126 | + if src_file in src_package['files']: |
127 | + return { |
128 | + 'ftype': ftype, |
129 | + 'sha256': sha256, |
130 | + 'path': stream_path, |
131 | + 'size': size, |
132 | + 'src_package': src_package['src_package'], |
133 | + 'src_version': src_package['src_version'], |
134 | + 'src_release': src_package['src_release'], |
135 | + } |
136 | + raise ValueError("%s not found in src_packages" % src_file) |
137 | + |
138 | + |
139 | +def archive_files(items, target): |
140 | + """Archive multiple files from a src_package into archive.tar.xz.""" |
141 | + archive_items = {} |
142 | + new_items = {} |
143 | + # Create a mapping of source packages and the files that came from them. |
144 | + for item in items.values(): |
145 | + key = "%(src_package)s-%(src_release)-%(src_version)" % item |
146 | + if archive_items.get(key) is None: |
147 | + archive_items[key] = { |
148 | + 'src_package': item['src_package'], |
149 | + 'src_release': item['src_release'], |
150 | + 'src_version': item['src_version'], |
151 | + 'files': [item['path']], |
152 | + } |
153 | + else: |
154 | + archive_items[key]['files'].append(item['path']) |
155 | + for item in archive_items.values(): |
156 | + stream_path = os.path.join( |
157 | + os.path.dirname(item['files'][0]), |
158 | + '%s.tar.xz' % item['src_package']) |
159 | + full_path = os.path.join(target, stream_path) |
160 | + tar = tarfile.open(full_path, 'w:xz') |
161 | + for f in item['files']: |
162 | + item_full_path = os.path.join(target, f) |
163 | + tar.add(item_full_path, os.path.basename(item_full_path)) |
164 | + os.remove(item_full_path) |
165 | + tar.close() |
166 | + sha256, size = get_file_info(full_path) |
167 | + new_items[item['src_package']] = { |
168 | + 'ftype': 'archive.tar.xz', |
169 | + 'sha256': sha256, |
170 | + 'path': stream_path, |
171 | + 'size': size, |
172 | + 'src_package': item['src_package'], |
173 | + 'src_release': item['src_release'], |
174 | + 'src_version': item['src_version'], |
175 | + } |
176 | + return new_items |
177 | + |
178 | + |
179 | def extract_files_from_packages( |
180 | - archive, packages, architecture, files, release, dest, |
181 | - grub_format=None, grub_config=None): |
182 | + archive, packages, architecture, files, release, target, path, |
183 | + grub_format=None, grub_config=None, grub_output=None): |
184 | tmp = tempfile.mkdtemp(prefix='maas-images-') |
185 | + src_packages = [] |
186 | for package in packages: |
187 | package = get_package(archive, package, architecture, release, tmp) |
188 | pkg_path = os.path.join(tmp, os.path.basename(package['Filename'])) |
189 | @@ -142,7 +228,23 @@ |
190 | sys.stderr.write('%s not found in archives!' % package) |
191 | sys.exit(1) |
192 | subprocess.check_output(['dpkg', '-x', pkg_path, tmp]) |
193 | - |
194 | + new_source_package = True |
195 | + for src_package in src_packages: |
196 | + if src_package['src_package'] == package['Source']: |
197 | + new_source_package = False |
198 | + src_package['files'] += package['files'] |
199 | + if new_source_package: |
200 | + # Some source packages include the package version in the source |
201 | + # name. Only take the name, not the version. |
202 | + src_package = package['Source'].split(' ')[0] |
203 | + src_packages.append({ |
204 | + 'src_package': src_package, |
205 | + 'src_version': package['Version'], |
206 | + 'src_release': release, |
207 | + 'files': package['files'], |
208 | + }) |
209 | + dest = os.path.join(target, path) |
210 | + items = {} |
211 | if grub_format is None: |
212 | for i in files: |
213 | if '*' in i or '?' in i: |
214 | @@ -150,20 +252,38 @@ |
215 | src = "%s/%s" % (tmp, i) |
216 | unglobbed_files = glob.glob(src) |
217 | for f in unglobbed_files: |
218 | - dest_file = "%s/%s" % (dest, os.path.basename(f)) |
219 | + basename = os.path.basename(f) |
220 | + dest_file = "%s/%s" % (dest, basename) |
221 | + stream_path = "%s/%s" % (path, basename) |
222 | shutil.copyfile(f, dest_file) |
223 | + pkg_file = f[len(tmp):] |
224 | + while pkg_file.startswith('/'): |
225 | + pkg_file = pkg_file[1:] |
226 | + items[basename] = make_item( |
227 | + 'bootloader', pkg_file, dest_file, stream_path, |
228 | + src_packages) |
229 | elif ',' in i: |
230 | # Copy the a file from the package using a new name |
231 | src_file, dest_file = i.split(',') |
232 | - src_file = "%s/%s" % (tmp, src_file.strip()) |
233 | - dest_file = "%s/%s" % (dest, dest_file.strip()) |
234 | - shutil.copyfile(src_file, dest_file) |
235 | + dest_file = dest_file.strip() |
236 | + full_src_file_path = "%s/%s" % (tmp, src_file.strip()) |
237 | + stream_path = "%s/%s" % (path, dest_file) |
238 | + full_dest_file_path = "%s/%s" % (dest, dest_file) |
239 | + shutil.copyfile(full_src_file_path, full_dest_file_path) |
240 | + items[dest_file] = make_item( |
241 | + 'bootloader', src_file, full_dest_file_path, stream_path, |
242 | + src_packages) |
243 | else: |
244 | # Straight copy |
245 | + basename = os.path.basename(i) |
246 | src_file = "%s/%s" % (tmp, i) |
247 | - dest_file = "%s/%s" % (dest, os.path.basename(src_file)) |
248 | + dest_file = "%s/%s" % (dest, basename) |
249 | + stream_path = "%s/%s" % (path, basename) |
250 | shutil.copyfile(src_file, dest_file) |
251 | + items[basename] = make_item( |
252 | + 'bootloader', i, dest_file, stream_path, src_packages) |
253 | else: |
254 | + dest = os.path.join(dest, grub_output) |
255 | # You can only tell grub to use modules from one directory |
256 | modules_path = "%s/%s" % (tmp, files[0]) |
257 | modules = [] |
258 | @@ -189,4 +309,9 @@ |
259 | '-O', grub_format, |
260 | '-d', modules_path, |
261 | ] + modules) |
262 | + basename = os.path.basename(dest) |
263 | + stream_path = "%s/%s" % (path, basename) |
264 | + items[basename] = make_item( |
265 | + 'bootloader', files[0], dest, stream_path, src_packages) |
266 | shutil.rmtree(tmp) |
267 | + return archive_files(items, target) |
268 | |
269 | === modified file 'meph2/commands/meph2_util.py' |
270 | --- meph2/commands/meph2_util.py 2016-08-01 19:15:59 +0000 |
271 | +++ meph2/commands/meph2_util.py 2016-08-24 20:40:59 +0000 |
272 | @@ -1,7 +1,7 @@ |
273 | #!/usr/bin/python3 |
274 | |
275 | import argparse |
276 | -import glob |
277 | +from datetime import datetime |
278 | import copy |
279 | import os |
280 | from functools import partial |
281 | @@ -176,9 +176,13 @@ |
282 | (pedigree, sutil.products_exdata( |
283 | src, pedigree, include_top=False, |
284 | insert_fieldnames=False)),) |
285 | + |
286 | return super(BareMirrorWriter, self).insert_item( |
287 | data, src, target, pedigree, contentsource) |
288 | |
289 | + def remove_item(self, data, src, target, pedigree): |
290 | + return |
291 | + |
292 | def remove_version(self, data, src, target, pedigree): |
293 | # sync doesnt filter on things to be removed, so |
294 | # we have to do that here. |
295 | @@ -187,9 +191,6 @@ |
296 | |
297 | self.removed_versions.append(pedigree) |
298 | |
299 | - def remove_item(self, data, src, target, pedigree): |
300 | - return |
301 | - |
302 | def insert_products(self, path, target, content): |
303 | # insert_item and insert_products would not be strictly necessary |
304 | # they're here, though, to keep a list of those things appended. |
305 | @@ -231,7 +232,9 @@ |
306 | |
307 | sutil.products_condense( |
308 | self.tproducts, |
309 | - sticky=['di_version', 'kpackage', 'sha256', 'md5', 'path']) |
310 | + sticky=[ |
311 | + 'di_version', 'kpackage', 'sha256', 'md5', 'path', 'ftype', |
312 | + 'src_package', 'src_version', 'src_release']) |
313 | |
314 | self.tproducts['updated'] = sutil.timestamp() |
315 | |
316 | @@ -525,100 +528,91 @@ |
317 | } |
318 | |
319 | |
320 | -def get_file_info(f): |
321 | - size = 0 |
322 | - sha256 = hashlib.sha256() |
323 | - with open(f, 'rb') as f: |
324 | - for chunk in iter(lambda: f.read(2**15), b''): |
325 | - sha256.update(chunk) |
326 | - size += len(chunk) |
327 | - return sha256.hexdigest(), size |
328 | - |
329 | - |
330 | def import_bootloaders(args, product_tree, cfgdata): |
331 | - for bootloader in cfgdata['bootloaders']: |
332 | + for firmware_platform in cfgdata['bootloaders']: |
333 | product_id = cfgdata['product_id'].format( |
334 | - bootloader=bootloader['bootloader']) |
335 | - package = get_package( |
336 | - bootloader['archive'], bootloader['packages'][0], |
337 | - bootloader['arch'], bootloader['release']) |
338 | - |
339 | - if ( |
340 | - product_id in product_tree['products'] and |
341 | - package['Version'] in product_tree['products'][product_id][ |
342 | - 'versions']): |
343 | + os=firmware_platform['os'], |
344 | + firmware_platform=firmware_platform['firmware-platform'], |
345 | + arch=firmware_platform['arch']) |
346 | + # Compile a list of the latest packages in the archive this bootloader |
347 | + # pulls files from |
348 | + src_packages = {} |
349 | + for package in firmware_platform['packages']: |
350 | + package_info = get_package( |
351 | + firmware_platform['archive'], package, |
352 | + firmware_platform['arch'], firmware_platform['release']) |
353 | + # Some source packages include the package version in the source |
354 | + # name. Only take the name, not the version. |
355 | + src_package_name = package_info['Source'].split(' ')[0] |
356 | + src_packages[src_package_name] = { |
357 | + 'src_version': package_info['Version'], |
358 | + 'src_release': firmware_platform['release'], |
359 | + 'found': False, |
360 | + } |
361 | + # Check if the bootloader has been built from the latest version of |
362 | + # the packages in the archive |
363 | + if product_id in product_tree['products']: |
364 | + versions = product_tree['products'][product_id]['versions'] |
365 | + for data in versions.values(): |
366 | + for item in data['items'].values(): |
367 | + src_package = src_packages.get(item['src_package']) |
368 | + if ( |
369 | + src_package is not None and |
370 | + src_package['src_version'] == item['src_version'] |
371 | + and |
372 | + src_package['src_release'] == item['src_release']): |
373 | + src_packages[item['src_package']]['found'] = True |
374 | + bootloader_uptodate = True |
375 | + for src_package in src_packages.values(): |
376 | + if not src_package['found']: |
377 | + bootloader_uptodate = False |
378 | + # Bootloader built from the latest packages already in stream |
379 | + if bootloader_uptodate: |
380 | print( |
381 | - "Product %s at version %s exists, skipping" % ( |
382 | - product_id, package['Version'])) |
383 | + "Product %s built from the latest package set, skipping" |
384 | + % product_id) |
385 | continue |
386 | + # Find an unused version |
387 | + today = datetime.utcnow().strftime('%Y%m%d') |
388 | + point = 0 |
389 | + while True: |
390 | + version = "%s.%d" % (today, point) |
391 | + products = product_tree['products'] |
392 | + if ( |
393 | + product_id not in products or |
394 | + version not in products[product_id]['versions'].keys()): |
395 | + break |
396 | + point += 1 |
397 | if product_tree['products'].get(product_id) is None: |
398 | print("Creating new product %s" % product_id) |
399 | product_tree['products'][product_id] = { |
400 | 'label': 'daily', |
401 | - 'arch': bootloader['arch'], |
402 | - 'subarch': 'generic', |
403 | - 'subarches': 'generic', |
404 | - 'os': 'bootloader', |
405 | - 'release': bootloader['bootloader'], |
406 | + 'arch': firmware_platform['arch'], |
407 | + 'arches': firmware_platform['arches'], |
408 | + 'os': firmware_platform['os'], |
409 | + 'bootloader-type': firmware_platform['firmware-platform'], |
410 | 'versions': {}, |
411 | } |
412 | path = os.path.join( |
413 | - 'bootloaders', bootloader['bootloader'], bootloader['arch'], |
414 | - package['Version']) |
415 | + 'bootloaders', firmware_platform['firmware-platform'], |
416 | + firmware_platform['arch'], version) |
417 | dest = os.path.join(args.target, path) |
418 | os.makedirs(dest) |
419 | - grub_format = bootloader.get('grub_format') |
420 | + grub_format = firmware_platform.get('grub_format') |
421 | if grub_format is not None: |
422 | - dest = os.path.join(dest, bootloader['grub_output']) |
423 | + dest = os.path.join(dest, firmware_platform['grub_output']) |
424 | print( |
425 | "Downloading and creating %s version %s" % ( |
426 | - product_id, package['Version'])) |
427 | - extract_files_from_packages( |
428 | - bootloader['archive'], bootloader['packages'], |
429 | - bootloader['arch'], bootloader['files'], bootloader['release'], |
430 | - dest, grub_format, bootloader.get('grub_config')) |
431 | - if grub_format is not None: |
432 | - sha256, size = get_file_info(dest) |
433 | - product_tree['products'][product_id]['versions'][ |
434 | - package['Version']] = { |
435 | - 'items': { |
436 | - bootloader['grub_output']: { |
437 | - 'ftype': 'bootloader', |
438 | - 'sha256': sha256, |
439 | - 'path': os.path.join( |
440 | - path, bootloader['grub_output']), |
441 | - 'size': size, |
442 | - } |
443 | - } |
444 | - } |
445 | - else: |
446 | - items = {} |
447 | - for i in bootloader['files']: |
448 | - basename = os.path.basename(i) |
449 | - dest_file = os.path.join(dest, basename) |
450 | - if '*' in dest_file or '?' in dest_file: |
451 | - # Process multiple files copied with a wildcard |
452 | - unglobbed_files = glob.glob(dest_file) |
453 | - elif ',' in dest_file: |
454 | - # If we're renaming the file from the package use the new |
455 | - # name. |
456 | - _, basename = i.split(',') |
457 | - basename = basename.strip() |
458 | - dest_file = os.path.join(dest, basename) |
459 | - unglobbed_files = [dest_file] |
460 | - else: |
461 | - unglobbed_files = [dest_file] |
462 | - for f in unglobbed_files: |
463 | - basename = os.path.basename(f) |
464 | - sha256, size = get_file_info(f) |
465 | - items[basename] = { |
466 | - 'ftype': 'bootloader', |
467 | - 'sha256': sha256, |
468 | - 'path': os.path.join(path, basename), |
469 | - 'size': size, |
470 | - } |
471 | - product_tree['products'][product_id]['versions'][ |
472 | - package['Version']] = {'items': items} |
473 | + product_id, version)) |
474 | + items = extract_files_from_packages( |
475 | + firmware_platform['archive'], firmware_platform['packages'], |
476 | + firmware_platform['arch'], firmware_platform['files'], |
477 | + firmware_platform['release'], args.target, path, grub_format, |
478 | + firmware_platform.get('grub_config'), |
479 | + firmware_platform.get('grub_output')) |
480 | + product_tree['products'][product_id]['versions'][version] = { |
481 | + 'items': items |
482 | + } |
483 | |
484 | |
485 | def main_import(args): |
As discussed on IRC pull the bootloaders from Xenial. The sample output has been updated.