Merge lp:~ltrager/maas-images/bootloaders_part2 into lp:maas-images

[Lee Trager]

Update bootloader format as per discussion in https://code.launchpad.net/~ltrager/maas-images/rename_squashfs/+merge/302353

Sample output:
http://paste.ubuntu.com/23083035/

[lp:~ltrager/maas-images/bootloaders_part2]

324. By Lee Trager on 2016-08-23

Put os in bootloader product name

325. By Lee Trager on 2016-08-23

Use bootloaders from Xenial

[Lee Trager (ltrager)]

As discussed on IRC pull the bootloaders from Xenial. The sample output has been updated.

[Scott Moser (smoser)]

the data looks good.

i have one nit pick in the conf/bootloaders.yaml
I think better to have:

   files:
    - usr/lib/grub/arm64-efi

than

   files:
    - /usr/lib/grub/arm64-efi

just to make it clear that the files are coming from the package itself and not the isntalled system, and also to avoid the pitfall of:
  os.path.join("/home/smoser", "/etc/passwd") == "/etc/passwd"

review: Approve

[lp:~ltrager/maas-images/bootloaders_part2]

326. By Lee Trager on 2016-08-24

Remove leading / from bootloader files

=== modified file 'conf/bootloaders.yaml'
--- conf/bootloaders.yaml	2016-08-23 15:17:16 +0000
+++ conf/bootloaders.yaml	2016-08-24 20:40:59 +0000
@@ -1,35 +1,41 @@
-product_id: "com.ubuntu.maas.daily:1:bootloaders-bases:{bootloader}"
-content_id: "com.ubuntu.maas:daily:1:bootloaders-bases-download"
+product_id: "com.ubuntu.maas.daily:1:{os}:{firmware_platform}:{arch}"
+content_id: "com.ubuntu.maas:daily:1:bootloader-download"
 
 bootloaders:
- - bootloader: pxe
+ - firmware-platform: pxe
    packages:
     - pxelinux
     - syslinux-common
-   arch: amd64
+   arch: i386
+   arches: i386,amd64
    archive: http://archive.ubuntu.com/ubuntu
    release: xenial
+   os: pxelinux
    files:
-    - /usr/lib/PXELINUX/pxelinux.0
-    - /usr/lib/syslinux/modules/bios/*.c32
- - bootloader: uefi
+    - usr/lib/PXELINUX/pxelinux.0
+    - usr/lib/syslinux/modules/bios/*.c32
+ - firmware-platform: uefi
    packages:
     - shim-signed
     - grub-efi-amd64-signed
    arch: amd64
+   arches: amd64
    archive: http://archive.ubuntu.com/ubuntu
    release: xenial
+   os: grub-efi-signed
    files:
-    - /usr/lib/shim/shim.efi.signed, bootx64.efi
-    - /usr/lib/grub/x86_64-efi-signed/grubnetx64.efi.signed, grubx64.efi
- - bootloader: uefi-arm64
+    - usr/lib/shim/shim.efi.signed, bootx64.efi
+    - usr/lib/grub/x86_64-efi-signed/grubnetx64.efi.signed, grubx64.efi
+ - firmware-platform: uefi
    packages:
     - grub-efi-arm64-bin
    arch: arm64
+   arches: arm64
    archive: http://ports.ubuntu.com
    release: xenial
+   os: grub-efi
    files:
-    - /usr/lib/grub/arm64-efi/
+    - usr/lib/grub/arm64-efi/
    grub_format: arm64-efi
    grub_output: grubaa64.efi
    grub_config: |
@@ -41,14 +47,16 @@
       # Failed to load based on MAC address.
       # Load arm64 by default, UEFI only supported by 64-bit
       configfile (pxe)/grub/grub.cfg-default-arm64
- - bootloader: powerkvm
+ - firmware-platform: open-firmware
    packages:
     - grub-ieee1275-bin
    arch: ppc64el
+   arches: ppc64el,ppc64
    archive: http://ports.ubuntu.com
    release: xenial
+   os: grub-ieee1275
    files:
-    - /usr/lib/grub/powerpc-ieee1275/
+    - usr/lib/grub/powerpc-ieee1275/
    grub_format: powerpc-ieee1275
    grub_output: bootppc64.bin
    grub_config: |
 
=== modified file 'meph2/commands/dpkg.py'
--- meph2/commands/dpkg.py	2016-08-03 13:00:03 +0000
+++ meph2/commands/dpkg.py	2016-08-24 20:40:59 +0000
@@ -6,6 +6,7 @@
 import os
 import re
 import sys
+import tarfile
 import tempfile
 import glob
 
@@ -128,13 +129,98 @@
         pkg_path = os.path.join(dest, os.path.basename(package['Filename']))
         with open(pkg_path, 'wb') as stream:
             stream.write(pkg_data)
+        package['files'] = []
+        output = subprocess.check_output(['dpkg', '-c', pkg_path])
+        for line in output.decode('utf-8').split('\n'):
+            # The file is the last column in the list.
+            file_info = line.split()
+            # Last line is just a newline
+            if len(file_info) == 0:
+                continue
+            if file_info[-1].startswith('./'):
+                # Remove leading './' if it exists
+                f = file_info[-1][2:]
+            elif file_info[-1].startswith('/'):
+                # Removing leading '/' if it exists
+                f = file_info[-1][1:]
+            else:
+                f = file_info[-1]
+            if f != '':
+                package['files'].append(f)
     return package
 
 
+def get_file_info(f):
+    size = 0
+    sha256 = hashlib.sha256()
+    with open(f, 'rb') as f:
+        for chunk in iter(lambda: f.read(2**15), b''):
+            sha256.update(chunk)
+            size += len(chunk)
+    return sha256.hexdigest(), size
+
+
+def make_item(ftype, src_file, dest_file, stream_path, src_packages):
+    sha256, size = get_file_info(dest_file)
+    for src_package in src_packages:
+        if src_file in src_package['files']:
+            return {
+                'ftype': ftype,
+                'sha256': sha256,
+                'path': stream_path,
+                'size': size,
+                'src_package': src_package['src_package'],
+                'src_version': src_package['src_version'],
+                'src_release': src_package['src_release'],
+            }
+    raise ValueError("%s not found in src_packages" % src_file)
+
+
+def archive_files(items, target):
+    """Archive multiple files from a src_package into archive.tar.xz."""
+    archive_items = {}
+    new_items = {}
+    # Create a mapping of source packages and the files that came from them.
+    for item in items.values():
+        key = "%(src_package)s-%(src_release)-%(src_version)" % item
+        if archive_items.get(key) is None:
+            archive_items[key] = {
+                'src_package': item['src_package'],
+                'src_release': item['src_release'],
+                'src_version': item['src_version'],
+                'files': [item['path']],
+            }
+        else:
+            archive_items[key]['files'].append(item['path'])
+    for item in archive_items.values():
+        stream_path = os.path.join(
+            os.path.dirname(item['files'][0]),
+            '%s.tar.xz' % item['src_package'])
+        full_path = os.path.join(target, stream_path)
+        tar = tarfile.open(full_path, 'w:xz')
+        for f in item['files']:
+            item_full_path = os.path.join(target, f)
+            tar.add(item_full_path, os.path.basename(item_full_path))
+            os.remove(item_full_path)
+        tar.close()
+        sha256, size = get_file_info(full_path)
+        new_items[item['src_package']] = {
+            'ftype': 'archive.tar.xz',
+            'sha256': sha256,
+            'path': stream_path,
+            'size': size,
+            'src_package': item['src_package'],
+            'src_release': item['src_release'],
+            'src_version': item['src_version'],
+        }
+    return new_items
+
+
 def extract_files_from_packages(
-        archive, packages, architecture, files, release, dest,
-        grub_format=None, grub_config=None):
+        archive, packages, architecture, files, release, target, path,
+        grub_format=None, grub_config=None, grub_output=None):
     tmp = tempfile.mkdtemp(prefix='maas-images-')
+    src_packages = []
     for package in packages:
         package = get_package(archive, package, architecture, release, tmp)
         pkg_path = os.path.join(tmp, os.path.basename(package['Filename']))
@@ -142,7 +228,23 @@
             sys.stderr.write('%s not found in archives!' % package)
             sys.exit(1)
         subprocess.check_output(['dpkg', '-x', pkg_path, tmp])
-
+        new_source_package = True
+        for src_package in src_packages:
+            if src_package['src_package'] == package['Source']:
+                new_source_package = False
+                src_package['files'] += package['files']
+        if new_source_package:
+            # Some source packages include the package version in the source
+            # name. Only take the name, not the version.
+            src_package = package['Source'].split(' ')[0]
+            src_packages.append({
+                'src_package': src_package,
+                'src_version': package['Version'],
+                'src_release': release,
+                'files': package['files'],
+            })
+    dest = os.path.join(target, path)
+    items = {}
     if grub_format is None:
         for i in files:
             if '*' in i or '?' in i:
@@ -150,20 +252,38 @@
                 src = "%s/%s" % (tmp, i)
                 unglobbed_files = glob.glob(src)
                 for f in unglobbed_files:
-                    dest_file = "%s/%s" % (dest, os.path.basename(f))
+                    basename = os.path.basename(f)
+                    dest_file = "%s/%s" % (dest, basename)
+                    stream_path = "%s/%s" % (path, basename)
                     shutil.copyfile(f, dest_file)
+                    pkg_file = f[len(tmp):]
+                    while pkg_file.startswith('/'):
+                        pkg_file = pkg_file[1:]
+                    items[basename] = make_item(
+                        'bootloader', pkg_file, dest_file, stream_path,
+                        src_packages)
             elif ',' in i:
                 # Copy the a file from the package using a new name
                 src_file, dest_file = i.split(',')
-                src_file = "%s/%s" % (tmp, src_file.strip())
-                dest_file = "%s/%s" % (dest, dest_file.strip())
-                shutil.copyfile(src_file, dest_file)
+                dest_file = dest_file.strip()
+                full_src_file_path = "%s/%s" % (tmp, src_file.strip())
+                stream_path = "%s/%s" % (path, dest_file)
+                full_dest_file_path = "%s/%s" % (dest, dest_file)
+                shutil.copyfile(full_src_file_path, full_dest_file_path)
+                items[dest_file] = make_item(
+                    'bootloader', src_file, full_dest_file_path, stream_path,
+                    src_packages)
             else:
                 # Straight copy
+                basename = os.path.basename(i)
                 src_file = "%s/%s" % (tmp,  i)
-                dest_file = "%s/%s" % (dest, os.path.basename(src_file))
+                dest_file = "%s/%s" % (dest, basename)
+                stream_path = "%s/%s" % (path, basename)
                 shutil.copyfile(src_file, dest_file)
+                items[basename] = make_item(
+                    'bootloader', i, dest_file, stream_path, src_packages)
     else:
+        dest = os.path.join(dest, grub_output)
         # You can only tell grub to use modules from one directory
         modules_path = "%s/%s" % (tmp, files[0])
         modules = []
@@ -189,4 +309,9 @@
                  '-O', grub_format,
                  '-d', modules_path,
                  ] + modules)
+        basename = os.path.basename(dest)
+        stream_path = "%s/%s" % (path, basename)
+        items[basename] = make_item(
+            'bootloader', files[0], dest, stream_path, src_packages)
     shutil.rmtree(tmp)
+    return archive_files(items, target)
 
=== modified file 'meph2/commands/meph2_util.py'
--- meph2/commands/meph2_util.py	2016-08-01 19:15:59 +0000
+++ meph2/commands/meph2_util.py	2016-08-24 20:40:59 +0000
@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
 import argparse
-import glob
+from datetime import datetime
 import copy
 import os
 from functools import partial
@@ -176,9 +176,13 @@
             (pedigree, sutil.products_exdata(
                 src, pedigree, include_top=False,
                 insert_fieldnames=False)),)
+
         return super(BareMirrorWriter, self).insert_item(
             data, src, target, pedigree, contentsource)
 
+    def remove_item(self, data, src, target, pedigree):
+        return
+
     def remove_version(self, data, src, target, pedigree):
         # sync doesnt filter on things to be removed, so
         # we have to do that here.
@@ -187,9 +191,6 @@
 
         self.removed_versions.append(pedigree)
 
-    def remove_item(self, data, src, target, pedigree):
-        return
-
     def insert_products(self, path, target, content):
         # insert_item and insert_products would not be strictly necessary
         # they're here, though, to keep a list of those things appended.
@@ -231,7 +232,9 @@
 
         sutil.products_condense(
             self.tproducts,
-            sticky=['di_version', 'kpackage', 'sha256', 'md5', 'path'])
+            sticky=[
+                'di_version', 'kpackage', 'sha256', 'md5', 'path', 'ftype',
+                'src_package', 'src_version', 'src_release'])
 
         self.tproducts['updated'] = sutil.timestamp()
 
@@ -525,100 +528,91 @@
                 }
 
 
-def get_file_info(f):
-    size = 0
-    sha256 = hashlib.sha256()
-    with open(f, 'rb') as f:
-        for chunk in iter(lambda: f.read(2**15), b''):
-            sha256.update(chunk)
-            size += len(chunk)
-    return sha256.hexdigest(), size
-
-
 def import_bootloaders(args, product_tree, cfgdata):
-    for bootloader in cfgdata['bootloaders']:
+    for firmware_platform in cfgdata['bootloaders']:
         product_id = cfgdata['product_id'].format(
-            bootloader=bootloader['bootloader'])
-        package = get_package(
-            bootloader['archive'], bootloader['packages'][0],
-            bootloader['arch'], bootloader['release'])
-
-        if (
-                product_id in product_tree['products'] and
-                package['Version'] in product_tree['products'][product_id][
-                    'versions']):
+            os=firmware_platform['os'],
+            firmware_platform=firmware_platform['firmware-platform'],
+            arch=firmware_platform['arch'])
+        # Compile a list of the latest packages in the archive this bootloader
+        # pulls files from
+        src_packages = {}
+        for package in firmware_platform['packages']:
+            package_info = get_package(
+                firmware_platform['archive'], package,
+                firmware_platform['arch'], firmware_platform['release'])
+            # Some source packages include the package version in the source
+            # name. Only take the name, not the version.
+            src_package_name = package_info['Source'].split(' ')[0]
+            src_packages[src_package_name] = {
+                'src_version': package_info['Version'],
+                'src_release': firmware_platform['release'],
+                'found': False,
+            }
+        # Check if the bootloader has been built from the latest version of
+        # the packages in the archive
+        if product_id in product_tree['products']:
+            versions = product_tree['products'][product_id]['versions']
+            for data in versions.values():
+                for item in data['items'].values():
+                    src_package = src_packages.get(item['src_package'])
+                    if (
+                            src_package is not None and
+                            src_package['src_version'] == item['src_version']
+                            and
+                            src_package['src_release'] == item['src_release']):
+                        src_packages[item['src_package']]['found'] = True
+        bootloader_uptodate = True
+        for src_package in src_packages.values():
+            if not src_package['found']:
+                bootloader_uptodate = False
+        # Bootloader built from the latest packages already in stream
+        if bootloader_uptodate:
             print(
-                "Product %s at version %s exists, skipping" % (
-                    product_id, package['Version']))
+                "Product %s built from the latest package set, skipping"
+                % product_id)
             continue
+        # Find an unused version
+        today = datetime.utcnow().strftime('%Y%m%d')
+        point = 0
+        while True:
+            version = "%s.%d" % (today, point)
+            products = product_tree['products']
+            if (
+                    product_id not in products or
+                    version not in products[product_id]['versions'].keys()):
+                break
+            point += 1
         if product_tree['products'].get(product_id) is None:
             print("Creating new product %s" % product_id)
             product_tree['products'][product_id] = {
                 'label': 'daily',
-                'arch': bootloader['arch'],
-                'subarch': 'generic',
-                'subarches': 'generic',
-                'os': 'bootloader',
-                'release': bootloader['bootloader'],
+                'arch': firmware_platform['arch'],
+                'arches': firmware_platform['arches'],
+                'os': firmware_platform['os'],
+                'bootloader-type': firmware_platform['firmware-platform'],
                 'versions': {},
                 }
         path = os.path.join(
-            'bootloaders', bootloader['bootloader'], bootloader['arch'],
-            package['Version'])
+            'bootloaders', firmware_platform['firmware-platform'],
+            firmware_platform['arch'], version)
         dest = os.path.join(args.target, path)
         os.makedirs(dest)
-        grub_format = bootloader.get('grub_format')
+        grub_format = firmware_platform.get('grub_format')
         if grub_format is not None:
-            dest = os.path.join(dest, bootloader['grub_output'])
+            dest = os.path.join(dest, firmware_platform['grub_output'])
         print(
             "Downloading and creating %s version %s" % (
-                product_id, package['Version']))
-        extract_files_from_packages(
-            bootloader['archive'], bootloader['packages'],
-            bootloader['arch'], bootloader['files'], bootloader['release'],
-            dest, grub_format, bootloader.get('grub_config'))
-        if grub_format is not None:
-            sha256, size = get_file_info(dest)
-            product_tree['products'][product_id]['versions'][
-                package['Version']] = {
-                'items': {
-                    bootloader['grub_output']: {
-                        'ftype': 'bootloader',
-                        'sha256': sha256,
-                        'path': os.path.join(
-                            path, bootloader['grub_output']),
-                        'size': size,
-                        }
-                    }
-                }
-        else:
-            items = {}
-            for i in bootloader['files']:
-                basename = os.path.basename(i)
-                dest_file = os.path.join(dest, basename)
-                if '*' in dest_file or '?' in dest_file:
-                    # Process multiple files copied with a wildcard
-                    unglobbed_files = glob.glob(dest_file)
-                elif ',' in dest_file:
-                    # If we're renaming the file from the package use the new
-                    # name.
-                    _, basename = i.split(',')
-                    basename = basename.strip()
-                    dest_file = os.path.join(dest, basename)
-                    unglobbed_files = [dest_file]
-                else:
-                    unglobbed_files = [dest_file]
-                for f in unglobbed_files:
-                    basename = os.path.basename(f)
-                    sha256, size = get_file_info(f)
-                    items[basename] = {
-                        'ftype': 'bootloader',
-                        'sha256': sha256,
-                        'path': os.path.join(path, basename),
-                        'size': size,
-                    }
-                product_tree['products'][product_id]['versions'][
-                    package['Version']] = {'items': items}
+                product_id, version))
+        items = extract_files_from_packages(
+            firmware_platform['archive'], firmware_platform['packages'],
+            firmware_platform['arch'], firmware_platform['files'],
+            firmware_platform['release'], args.target, path, grub_format,
+            firmware_platform.get('grub_config'),
+            firmware_platform.get('grub_output'))
+        product_tree['products'][product_id]['versions'][version] = {
+            'items': items
+        }
 
 
 def main_import(args):