Merge lp:~waveform/meta-release/ubuntu into lp:~ubuntu-core-dev/meta-release/ubuntu

[Dave Jones]

Corrects hash for core armhf image in raspi JSON; adds utility for keeping the raspi JSON up to date relatively easily.

[lp:~waveform/meta-release/ubuntu]

244. By Brian Murray on 2020-04-16

merge waveform's branch to Correct hash for core armhf image in raspi JSON; add utility for keeping the raspi JSON up to date relatively easily.

=== modified file 'raspi/os_list_imagingutility_ubuntu.json'
--- raspi/os_list_imagingutility_ubuntu.json	2020-04-15 15:31:27 +0000
+++ raspi/os_list_imagingutility_ubuntu.json	2020-04-16 17:10:22 +0000
@@ -8,7 +8,8 @@
             "extract_size": 2388864000,
             "extract_sha256": "4ae38d8fb9d13c52aace1b81c4973b9d059228a62bba313055129314c203dd0a",
             "image_download_size": 494466080,
-            "release_date": "2020-02-03"
+            "release_date": "2020-02-03",
+            "image_download_sha256": "0382a2b87b6937ff645f59f546120e3480f2d5a2c2945712bf373379b9e82f51"
         },
         {
             "name": "Ubuntu 18.04.4 (Pi 3/4)",
@@ -18,7 +19,8 @@
             "extract_size": 2624517120,
             "extract_sha256": "be04570b43c5253b4bfe4beed1d2e9c76e9113ea5e5b3dc004ba83c5b5313dbb",
             "image_download_size": 499792288,
-            "release_date": "2020-02-03"
+            "release_date": "2020-02-03",
+            "image_download_sha256": "f270d4a11fcef7f85ea77bc0642d1c6db2666ae734e9dcc4cb875a31c9f0dc57"
         },
         {
             "name": "Ubuntu 19.10.1 (Pi 2/3/4)",
@@ -28,7 +30,8 @@
             "extract_size": 2832030720,
             "extract_sha256": "7564bf108b958f848a41633c27241fa6b991207467a1f55b0d4451c8a4ad5920",
             "image_download_size": 642984640,
-            "release_date": "2019-12-03"
+            "release_date": "2019-12-03",
+            "image_download_sha256": "d40820ba3933554b9902dc560e8a98d20352ae49b8544e71cacc6fdcfd2c2405"
         },
         {
             "name": "Ubuntu 19.10.1 (Pi 3/4)",
@@ -38,17 +41,19 @@
             "extract_size": 3115115520,
             "extract_sha256": "24b184762a2ebdca0e7f8151d3ea84e82c5e0c4a7f5a85f24f12f1cd5b197560",
             "image_download_size": 662616972,
-            "release_date": "2019-12-03"
+            "release_date": "2019-12-03",
+            "image_download_sha256": "3fcae7490edebd35663cb30dcd7a6317b6b9601d0f59ed5caa3c20dfd936cbb1"
         },
         {
             "name": "Ubuntu Core 18 (Pi 2/3/4)",
             "description": "Ubuntu Core 18 32-bit IoT OS for armhf architectures",
             "url": "http://cdimage.ubuntu.com/ubuntu-core/18/stable/current/ubuntu-core-18-armhf+raspi.img.xz",
             "icon": "https://assets.ubuntu.com/v1/85a9de76-ubuntu-icon.svg",
-            "extract_size": 563676160,
-            "extract_sha256": "21fb0f570b5398ccce65e3c0d2dcca4033d0c78cd9d7636c90d89ae0ec513b0c",
+            "extract_size": 700517376,
+            "extract_sha256": "ca6968a7e1c1977ee740b3e29b5b74430fc9aec55b1e0310fae85dae1c1af8e7",
             "image_download_size": 315745444,
-            "release_date": "2020-04-06"
+            "release_date": "2020-04-06",
+            "image_download_sha256": "31682b959c06487307a8cb2eef41b33878d1f6547fd5d3eb0b24256f9bf88c8e"
         },
         {
             "name": "Ubuntu Core 18 (Pi 3/4)",
@@ -58,7 +63,8 @@
             "extract_size": 709346304,
             "extract_sha256": "d005e21d1df4f33f897eaabd7ec66dc1446ac0c537cc3a79dfd9825a174fa12e",
             "image_download_size": 321941660,
-            "release_date": "2020-04-06"
+            "release_date": "2020-04-06",
+            "image_download_sha256": "1c7029f506da40c233b0257b2b395ac54126716cc2aaa87d0a4bf724c216992d"
         }
     ]
-}
+}
\ No newline at end of file

=== added file 'refresh_os_list'
--- refresh_os_list	1970-01-01 00:00:00 +0000
+++ refresh_os_list	2020-04-16 17:10:22 +0000
@@ -0,0 +1,232 @@
+#!/usr/bin/python3
+
+import os
+import io
+import sys
+import json
+import gzip
+import bz2
+import lzma
+import hashlib
+import argparse
+import warnings
+import functools
+import datetime as dt
+from html.parser import HTMLParser
+from urllib.parse import urlsplit, urlunsplit
+from urllib.request import urlopen, Request
+from collections import namedtuple, OrderedDict
+
+
+def main(args=None):
+    """
+    A script for generating the JSON list required by the Raspberry Pi imaging
+    utility. Takes an existing JSON list as input and updates it by calculating
+    new image sizes and check-sums from the source files on the server. Note
+    that the existing JSON list can be partial. The only mandatory field in
+    each entry is the "url"; the script will fill out all other missing fields
+    (albeit with place-holders in the case of "name" and "description" which it
+    can't derive).
+    """
+    if args is None:
+        args = sys.argv[1:]
+    parser = argparse.ArgumentParser(description=main.__doc__)
+    parser.add_argument(
+        'input_file', type=argparse.FileType('r', encoding='utf-8'),
+        help="The partial JSON template to fill out")
+    parser.add_argument(
+        'output_file', nargs='?', type=argparse.FileType('w', encoding='utf-8'),
+        default=sys.stdout,
+        help="The output file to create; defaults to stdout")
+    parser.add_argument(
+        '-f', '--force', action='store_true',
+        help="Force the utility to refresh all images even if the release "
+        "date and download size have not changed in the index")
+    args = parser.parse_args(args)
+    try:
+        update_template(args.input_file, args.output_file, args.force)
+    except Exception as e:
+        if not int(os.environ.get('DEBUG', '0')):
+            print(str(e), file=sys.stderr)
+            sys.exit(1)
+        else:
+            raise
+
+
+def update_template(input_file, output_file, force=False):
+    template = json.load(input_file, object_pairs_hook=OrderedDict)
+    if not isinstance(template, dict):
+        raise ValueError('expected a JSON object in {}'.format(args.input_file))
+    if template.keys() != {'os_list'}:
+        raise ValueError('expected a single "os_list" entry')
+    if not all('url' in entry for entry in template['os_list']):
+        raise ValueError('all "os_list" entries must contain a "url" entry')
+
+    for entry in template['os_list']:
+        url = entry['url']
+        source = get_entry(url)
+        if 'icon' not in entry:
+            entry['icon'] = 'https://assets.ubuntu.com/v1/85a9de76-ubuntu-icon.svg'
+        if 'name' not in entry:
+            warnings.warn(Warning('Inserted placeholder entries; check output'))
+            entry['name'] = 'PLACEHOLDER'
+        if 'description' not in entry:
+            warnings.warn(Warning('Inserted placeholder entries; check output'))
+            entry['description'] = 'PLACEHOLDER'
+        update = (
+            force or
+            # image_download_sha256 is optional and may be missing, hence the
+            # unusual default here
+            entry.get('image_download_sha256', source.sha256) != source.sha256 or
+            entry.get('image_download_size', 0) != source.size or
+            dt.datetime.strptime(
+                entry.get('release_date', '1970-01-01'), '%Y-%m-%d'
+            ).date() != source.date
+        )
+        if update:
+            print('Updating {}'.format(source.name), file=sys.stderr)
+            with HashPassthru(urlopen(url)) as compressed_image:
+                decompresser = {
+                    'gz': gzip.open,
+                    'bz2': bz2.open,
+                    'xz': lambda fd: lzma.open(fd, format=lzma.FORMAT_XZ),
+                }[url.rsplit('.', 1)[1]]
+                with decompresser(compressed_image) as decompressed_image:
+                    cksum = hashlib.sha256()
+                    size = 0
+                    while True:
+                        buf = decompressed_image.read(65536)
+                        if not buf:
+                            break
+                        size += len(buf)
+                        cksum.update(buf)
+                    entry['extract_size'] = size
+                    entry['extract_sha256'] = cksum.hexdigest().lower()
+                if compressed_image.size != source.size:
+                    raise ValueError('Corrupted download; size check failed')
+                if compressed_image.cksum.hexdigest().lower() != source.sha256:
+                    raise ValueError('Corrupted download; SHA256 check failed')
+            entry['image_download_size'] = source.size
+            entry['release_date'] = source.date.strftime('%Y-%m-%d')
+        else:
+            print('Entry for {} is up to date'.format(source.name),
+                  file=sys.stderr)
+        # Always update image_download_sha256 to fill it out when missing
+        entry['image_download_sha256'] = source.sha256
+    json.dump(template, output_file, indent=4)
+
+
+class HashPassthru:
+    def __init__(self, stream):
+        self.stream = stream
+        self.cksum = hashlib.sha256()
+        self.size = 0
+
+    def read(self, n):
+        result = self.stream.read(n)
+        self.size += len(result)
+        self.cksum.update(result)
+        return result
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.stream.close()
+
+
+class TableParser(HTMLParser):
+    """
+    A subclass of :class:`html.parser.HTMLParser` used by :func:`get_entry` for
+    parsing the <table> out of a web-page representing a directory of OS
+    images.
+    """
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.state = 'html'
+        self.table = []
+
+    def handle_starttag(self, tag, attrs):
+        if self.state == 'html' and tag == 'table':
+            self.state = 'table'
+        elif self.state == 'table' and tag == 'tr':
+            self.state = 'tr'
+            self.table.append([])
+        elif self.state == 'tr' and tag in ('th', 'td'):
+            self.state = 'td'
+            self.table[-1].append(None)
+
+    def handle_data(self, data):
+        if self.state == 'td':
+            self.table[-1][-1] = data
+
+    def handle_endtag(self, tag):
+        if self.state == 'table' and tag == 'table':
+            self.state = 'html'
+        elif self.state == 'tr' and tag == 'tr':
+            self.state = 'table'
+        elif self.state == 'td' and tag in ('th', 'td'):
+            self.state = 'tr'
+
+
+IndexEntry = namedtuple('IndexEntry', ('name', 'date', 'sha256', 'size'))
+
+
+def get_entry(url):
+    """
+    Given the *url* of an image, returns an :class:`IndexEntry` named tuple
+    containing the name, generated date, SHA-256 check-sum, and file size.
+    """
+    split = urlsplit(url)
+    path, name = split.path.rsplit('/', 1)
+    index = urlunsplit(split._replace(path=path + '/'))
+    entry = get_index(index)[name]
+    if entry.size is None or entry.sha256 is None:
+        raise ValueError('unable to retrieve file-size or checksum for '
+                         '{}'.format(url))
+    return entry
+
+
[e-mail]_cache()
+def get_index(url):
+    """
+    Given the *url* of a directory containing images, returns a dict mapping
+    filenames to :class:`IndexEntry` named tuples.
+    """
+    parser = TableParser()
+    with urlopen(url) as page:
+        parser.feed(page.read().decode('utf-8'))
+    entries = {}
+    for row in parser.table:
+        try:
+            icon, name, date, size = row
+        except ValueError:
+            # Evidently not a file row
+            continue
+        name = name.strip()
+        try:
+            date = dt.datetime.strptime(date.strip(), '%Y-%m-%d %H:%M').date()
+        except ValueError:
+            # Evidently not a file row
+            continue
+        sha256 = None
+        request = Request(url + name, method='HEAD')
+        with urlopen(request) as head:
+            size = int(head.getheader('Content-Length', '0'))
+        entries[name] = IndexEntry(name, date, sha256, size)
+    if 'SHA256SUMS' in entries:
+        with urlopen(url + 'SHA256SUMS') as hashes:
+            for line in hashes:
+                cksum, name = line.decode('utf-8').strip().split(None, 1)
+                cksum = cksum.strip().lower()
+                if name.startswith('*'):
+                    name = name[1:]
+                try:
+                    entries[name] = entries[name]._replace(sha256=cksum)
+                except KeyError:
+                    pass
+    return entries
+
+
+if __name__ == '__main__':
+    main()