Merge lp:~smoser/maas-images/trunk.meph-import-out into lp:maas-images

[Scott Moser]

[lp:~smoser/maas-images/trunk.meph-import-out]

334. By Scott Moser on 2016-09-26

fix usage of contentsource

[Lee Trager (ltrager)]

Looks good, things were just moved around a bit. One thing I noticed when I checked this out is meph2-import doesn't have the executable bit set.

review: Approve

[lp:~smoser/maas-images/trunk.meph-import-out]

335. By Scott Moser on 2016-09-26

add -x to bin/meph2-import

[Scott Moser (smoser)]

fixed the meph2-import executable. that was fallout of just bzr diff | patch -p0
and such.

=== added file 'bin/meph2-import'
--- bin/meph2-import	1970-01-01 00:00:00 +0000
+++ bin/meph2-import	2016-09-26 19:23:59 +0000
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+
+def call_entry_point(name):
+    (istr, dot, ent) = name.rpartition('.')
+    try:
+        __import__(istr)
+    except ImportError:
+        # if that import failed, check dirname(__file__/..)
+        # to support ./bin/program with modules in .
+        _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+        sys.path.insert(0, _tdir)
+        try:
+            __import__(istr)
+        except ImportError as e:
+            sys.stderr.write("Unable to find %s: %s\n" % (name, e))
+            sys.exit(2)
+
+    sys.exit(getattr(sys.modules[istr], ent)())
+
+if __name__ == '__main__':
+    call_entry_point("meph2.commands.mimport.main")
+
+# vi: ts=4 expandtab syntax=python

=== added file 'meph2/commands/flags.py'
--- meph2/commands/flags.py	1970-01-01 00:00:00 +0000
+++ meph2/commands/flags.py	2016-09-26 19:23:59 +0000
@@ -0,0 +1,107 @@
+DEF_KEYRING = "/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg"
+
+LABELS = ('alpha1', 'alpha2', 'alpha3',
+          'beta1', 'beta2', 'beta3',
+          'rc', 'release')
+
+COMMON_ARGS = []
+COMMON_FLAGS = {
+    'dry-run': (('-n', '--dry-run'),
+                {'help': 'only report what would be done',
+                 'action': 'store_true', 'default': False}),
+    'no-sign': (('-u', '--no-sign'),
+                {'help': 'do not re-sign files',
+                 'action': 'store_true', 'default': False}),
+    'max': (('--max',),
+            {'help': 'keep at most N versions per product',
+             'default': 2, 'type': int}),
+    'orphan-data': (('orphan_data',), {'help': 'the orphan data file'}),
+    'src': (('src',), {'help': 'the source streams directory'}),
+    'target': (('target',), {'help': 'the target streams directory'}),
+    'data_d': (('data_d',),
+               {'help': ('the base data directory'
+                         '("path"s are relative to this)')}),
+    'keyring': (('--keyring',),
+                {'help': 'gpg keyring to check sjson',
+                 'default': DEF_KEYRING}),
+}
+
+SUBCOMMANDS = {
+    'insert': {
+        'help': 'add new items from one stream into another',
+        'opts': [
+            COMMON_FLAGS['dry-run'], COMMON_FLAGS['no-sign'],
+            COMMON_FLAGS['keyring'],
+            COMMON_FLAGS['src'], COMMON_FLAGS['target'],
+            ('filters', {'nargs': '*', 'default': []}),
+        ]
+    },
+    'import': {
+        'help': 'import an image from the specified config into a stream',
+        'opts': [
+            COMMON_FLAGS['no-sign'], COMMON_FLAGS['keyring'],
+            ('import_cfg', {'help':
+                            'The config file for the image to import.'}),
+            COMMON_FLAGS['target'],
+            ]
+    },
+    'merge': {
+        'help': 'merge two product streams together',
+        'opts': [
+            COMMON_FLAGS['no-sign'],
+            COMMON_FLAGS['src'], COMMON_FLAGS['target'],
+            ]
+    },
+    'promote': {
+        'help': 'promote a product/version from daily to release',
+        'opts': [
+            COMMON_FLAGS['dry-run'], COMMON_FLAGS['no-sign'],
+            COMMON_FLAGS['keyring'],
+            (('-l', '--label'),
+             {'default': 'release', 'choices': LABELS,
+              'help': 'the label to use'}),
+            (('--skip-file-copy',),
+             {'help': 'do not copy files, only metadata [TEST_ONLY]',
+              'action': 'store_true', 'default': False}),
+            COMMON_FLAGS['src'], COMMON_FLAGS['target'],
+            ('version', {'help': 'the version_id to promote.'}),
+            ('filters', {'nargs': '+', 'default': []}),
+        ]
+    },
+    'clean-md': {
+        'help': 'clean streams metadata only to keep "max" items',
+        'opts': [
+            COMMON_FLAGS['dry-run'], COMMON_FLAGS['no-sign'],
+            COMMON_FLAGS['keyring'],
+            ('max', {'type': int}), ('target', {}),
+            ('filters', {'nargs': '*', 'default': []}),
+        ]
+    },
+    'find-orphans': {
+        'help': 'find files in data_d not referenced in a "path"',
+        'opts': [
+            COMMON_FLAGS['orphan-data'], COMMON_FLAGS['data_d'],
+            COMMON_FLAGS['keyring'],
+            ('streams_dirs', {'nargs': '*', 'default': []}),
+        ],
+    },
+    'reap-orphans': {
+        'help': 'reap orphans listed in orphan-data from data_d',
+        'opts': [
+            COMMON_FLAGS['orphan-data'], COMMON_FLAGS['dry-run'],
+            COMMON_FLAGS['data_d'],
+            ('--older', {'default': '3d',
+                         'help': ('only remove files orphaned longer than'
+                                  'this. if no unit given, default is days.')
+                         }),
+        ],
+    },
+    'sign': {
+        'help': 'Regenerate index.json and sign the stream',
+        'opts': [
+            COMMON_FLAGS['data_d'], COMMON_FLAGS['no-sign'],
+        ],
+    },
+}
+
+# vi: ts=4 expandtab syntax=python

=== modified file 'meph2/commands/meph2_util.py'
--- meph2/commands/meph2_util.py	2016-09-23 16:19:38 +0000
+++ meph2/commands/meph2_util.py	2016-09-26 19:23:59 +0000
@@ -1,139 +1,21 @@
 #!/usr/bin/python3
 
 import argparse
-from datetime import datetime
 import copy
 import os
 from functools import partial
-import hashlib
-import re
 import shutil
 import sys
-import subprocess
-import yaml
 
 from meph2 import util
-from meph2.url_helper import geturl_text
-from meph2.commands.dpkg import (
-    get_package,
-    extract_files_from_packages,
-)
+from meph2.commands.flags import COMMON_ARGS, SUBCOMMANDS
 
 from simplestreams import (
-    contentsource,
     filters,
     mirrors,
     util as sutil,
     objectstores)
 
-DEF_KEYRING = "/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg"
-
-STREAMS_D = "streams/v1/"
-
-LABELS = ('alpha1', 'alpha2', 'alpha3',
-          'beta1', 'beta2', 'beta3',
-          'rc', 'release')
-
-COMMON_ARGS = []
-COMMON_FLAGS = {
-    'dry-run': (('-n', '--dry-run'),
-                {'help': 'only report what would be done',
-                 'action': 'store_true', 'default': False}),
-    'no-sign': (('-u', '--no-sign'),
-                {'help': 'do not re-sign files',
-                 'action': 'store_true', 'default': False}),
-    'max': (('--max',),
-            {'help': 'keep at most N versions per product',
-             'default': 2, 'type': int}),
-    'orphan-data': (('orphan_data',), {'help': 'the orphan data file'}),
-    'src': (('src',), {'help': 'the source streams directory'}),
-    'target': (('target',), {'help': 'the target streams directory'}),
-    'data_d': (('data_d',),
-               {'help': ('the base data directory'
-                         '("path"s are relative to this)')}),
-    'keyring': (('--keyring',),
-                {'help': 'gpg keyring to check sjson',
-                 'default': DEF_KEYRING}),
-}
-
-SUBCOMMANDS = {
-    'insert': {
-        'help': 'add new items from one stream into another',
-        'opts': [
-            COMMON_FLAGS['dry-run'], COMMON_FLAGS['no-sign'],
-            COMMON_FLAGS['keyring'],
-            COMMON_FLAGS['src'], COMMON_FLAGS['target'],
-            ('filters', {'nargs': '*', 'default': []}),
-        ]
-    },
-    'import': {
-        'help': 'import an image from the specified config into a stream',
-        'opts': [
-            COMMON_FLAGS['no-sign'], COMMON_FLAGS['keyring'],
-            ('import_cfg', {'help':
-                            'The config file for the image to import.'}),
-            COMMON_FLAGS['target'],
-            ]
-    },
-    'merge': {
-        'help': 'merge two product streams together',
-        'opts': [
-            COMMON_FLAGS['no-sign'],
-            COMMON_FLAGS['src'], COMMON_FLAGS['target'],
-            ]
-    },
-    'promote': {
-        'help': 'promote a product/version from daily to release',
-        'opts': [
-            COMMON_FLAGS['dry-run'], COMMON_FLAGS['no-sign'],
-            COMMON_FLAGS['keyring'],
-            (('-l', '--label'),
-             {'default': 'release', 'choices': LABELS,
-              'help': 'the label to use'}),
-            (('--skip-file-copy',),
-             {'help': 'do not copy files, only metadata [TEST_ONLY]',
-              'action': 'store_true', 'default': False}),
-            COMMON_FLAGS['src'], COMMON_FLAGS['target'],
-            ('version', {'help': 'the version_id to promote.'}),
-            ('filters', {'nargs': '+', 'default': []}),
-        ]
-    },
-    'clean-md': {
-        'help': 'clean streams metadata only to keep "max" items',
-        'opts': [
-            COMMON_FLAGS['dry-run'], COMMON_FLAGS['no-sign'],
-            COMMON_FLAGS['keyring'],
-            ('max', {'type': int}), ('target', {}),
-            ('filters', {'nargs': '*', 'default': []}),
-        ]
-    },
-    'find-orphans': {
-        'help': 'find files in data_d not referenced in a "path"',
-        'opts': [
-            COMMON_FLAGS['orphan-data'], COMMON_FLAGS['data_d'],
-            COMMON_FLAGS['keyring'],
-            ('streams_dirs', {'nargs': '*', 'default': []}),
-        ],
-    },
-    'reap-orphans': {
-        'help': 'reap orphans listed in orphan-data from data_d',
-        'opts': [
-            COMMON_FLAGS['orphan-data'], COMMON_FLAGS['dry-run'],
-            COMMON_FLAGS['data_d'],
-            ('--older', {'default': '3d',
-                         'help': ('only remove files orphaned longer than'
-                                  'this. if no unit given, default is days.')
-                         }),
-        ],
-    },
-    'sign': {
-        'help': 'Regenerate index.json and sign the stream',
-        'opts': [
-            COMMON_FLAGS['data_d'], COMMON_FLAGS['no-sign'],
-        ],
-    },
-}
-
 
 class BareMirrorWriter(mirrors.ObjectFilterMirror):
     # this explicitly avoids reference counting and .data/ storage
@@ -332,109 +214,6 @@
         self.removed_versions.append((self.tcontent_id, pedigree,))
 
 
-def get_sha256_meta_images(url):
-    """ Given a URL to a SHA256SUM file return a dictionary of filenames and
-        SHA256 checksums keyed off the file version found as a date string in
-        the filename. This is used in cases where simplestream data isn't
-        avalible.
-    """
-    ret = dict()
-    content = geturl_text(url)
-    # http://cloud.centos.org/centos/ contains images using two version
-    # strings. The first is only used on older images and uses the format
-    # YYYYMMDD_XX. The second is used on images generated monthly using the
-    # format YYMM. We know the second format is referencing the year and month
-    # by looking at the timestamp of each image.
-    prog = re.compile('([\d]{8}(_[\d]+))|(\d{4})')
-
-    for i in content.split('\n'):
-        try:
-            sha256, img_name = i.split()
-        except ValueError:
-            continue
-        if (not img_name.endswith('qcow2.xz') and
-                not img_name.endswith('qcow2')):
-            continue
-        m = prog.search(img_name)
-        if m is None:
-            continue
-        img_version = m.group(0)
-
-        # Turn the short version string into a long version string so that MAAS
-        # uses the latest version, not the longest
-        if len(img_version) == 4:
-            img_version = "20%s01_01" % img_version
-
-        # Prefer compressed image over uncompressed
-        if (img_version in ret and
-                ret[img_version]['img_name'].endswith('qcow2.xz')):
-            continue
-        ret[img_version] = {
-            'img_name': img_name,
-            'sha256': sha256,
-            }
-    return ret
-
-
-def import_qcow2(url, expected_sha256, out, curtin_files=None):
-    """ Call the maas-qcow2targz script to convert a qcow2 or qcow2.xz file at
-        a given URL or local path. Return the SHA256SUM of the outputted file.
-    """
-    # Assume maas-qcow2targz is in the path
-    qcow2targz_cmd = ["maas-qcow2targz", url, expected_sha256, out]
-    if curtin_files:
-        curtin_path = os.path.join(
-            os.path.dirname(__file__), "..", "..", "curtin")
-        qcow2targz_cmd.append(curtin_files.format(curtin_path=curtin_path))
-    proc = subprocess.Popen(qcow2targz_cmd)
-    proc.communicate()
-    if proc.wait() != 0:
-        raise subprocess.CalledProcessError(
-            cmd=qcow2targz_cmd, returncode=proc.returncode)
-
-    sha256 = hashlib.sha256()
-    with open(out, 'rb') as fp:
-        while True:
-            chunk = fp.read(2**20)
-            if not chunk:
-                break
-            sha256.update(chunk)
-    return sha256.hexdigest()
-
-
-def load_product_streams(src):
-    index_path = os.path.join(src, STREAMS_D, "index.json")
-    if not os.path.exists(index_path):
-        return []
-    with contentsource.UrlContentSource(index_path) as tcs:
-        index = sutil.load_content(tcs.read())
-    return [product['path'] for product in index['index'].values()]
-
-
-def load_products(path, product_streams):
-    products = {}
-    for product_stream in product_streams:
-        product_stream_path = os.path.join(path, product_stream)
-        if os.path.exists(product_stream_path):
-            with contentsource.UrlContentSource(
-                    product_stream_path) as tcs:
-                product_listing = sutil.load_content(tcs.read())
-                products.update(product_listing['products'])
-    return products
-
-
-def gen_index_and_sign(data_d, sign=True):
-    md_d = os.path.join(data_d, "streams", "v1")
-    if not os.path.exists(md_d):
-        os.makedirs(md_d)
-    index = util.create_index(md_d, files=None)
-    with open(os.path.join(md_d, "index.json"), "wb") as fp:
-        fp.write(util.dump_data(index))
-
-    if sign:
-        util.sign_streams_d(md_d)
-
-
 def main_insert(args):
     (src_url, src_path) = sutil.path_from_mirror_url(args.src, None)
     filter_list = filters.get_filters(args.filters)
@@ -461,205 +240,15 @@
     tmirror = InsertBareMirrorWriter(config=mirror_config, objectstore=tstore)
     tmirror.sync(smirror, src_path)
 
-    gen_index_and_sign(args.target, sign=not args.no_sign)
+    util.gen_index_and_sign(args.target, sign=not args.no_sign)
     return 0
 
 
-def import_sha256(args, product_tree, cfgdata):
-    for (release, release_info) in cfgdata['versions'].items():
-        if 'arch' in release_info:
-            arch = release_info['arch']
-        else:
-            arch = cfgdata['arch']
-        if 'os' in release_info:
-            os_name = release_info['os']
-        else:
-            os_name = cfgdata['os']
-        if 'path_version' in release_info:
-            path_version = release_info['path_version']
-        else:
-            path_version = release_info['version']
-        product_id = cfgdata['product_id'].format(
-            version=release_info['version'], arch=arch)
-        url = cfgdata['sha256_meta_data_path'].format(version=path_version)
-        images = get_sha256_meta_images(url)
-        base_url = os.path.dirname(url)
-
-        if product_tree['products'].get(product_id) is None:
-            print("Creating new product %s" % product_id)
-            product_tree['products'][product_id] = {
-                'subarches': 'generic',
-                'label': 'daily',
-                'subarch': 'generic',
-                'arch': arch,
-                'os': os_name,
-                'version': release_info['version'],
-                'release': release,
-                'versions': {},
-            }
-
-        for (image, image_info) in images.items():
-            if (
-                    product_id in product_tree['products'] and
-                    image in product_tree['products'][product_id]['versions']):
-                print(
-                    "Product %s at version %s exists, skipping" % (
-                        product_id, image))
-                continue
-            print(
-                "Downloading and creating %s version %s" % (
-                    (product_id, image)))
-            image_path = '/'.join([release, arch, image, 'root-tgz'])
-            real_image_path = os.path.join(
-                os.path.realpath(args.target), image_path)
-            sha256 = import_qcow2(
-                '/'.join([base_url, image_info['img_name']]),
-                image_info['sha256'], real_image_path,
-                release_info.get('curtin_files'))
-            product_tree['products'][product_id]['versions'][image] = {
-                'items': {
-                    'root-image.gz': {
-                        'ftype': 'root-tgz',
-                        'sha256': sha256,
-                        'path': image_path,
-                        'size': os.path.getsize(real_image_path),
-                        }
-                    }
-                }
-
-
-def import_bootloaders(args, product_tree, cfgdata):
-    for firmware_platform in cfgdata['bootloaders']:
-        product_id = cfgdata['product_id'].format(
-            os=firmware_platform['os'],
-            firmware_platform=firmware_platform['firmware-platform'],
-            arch=firmware_platform['arch'])
-        # Compile a list of the latest packages in the archive this bootloader
-        # pulls files from
-        src_packages = {}
-        for package in firmware_platform['packages']:
-            package_info = get_package(
-                firmware_platform['archive'], package,
-                firmware_platform['arch'], firmware_platform['release'])
-            # Some source packages include the package version in the source
-            # name. Only take the name, not the version.
-            src_package_name = package_info['Source'].split(' ')[0]
-            src_packages[src_package_name] = {
-                'src_version': package_info['Version'],
-                'src_release': firmware_platform['release'],
-                'found': False,
-            }
-        # Check if the bootloader has been built from the latest version of
-        # the packages in the archive
-        if product_id in product_tree['products']:
-            versions = product_tree['products'][product_id]['versions']
-            for data in versions.values():
-                for item in data['items'].values():
-                    src_package = src_packages.get(item['src_package'])
-                    if (
-                            src_package is not None and
-                            src_package['src_version'] == item['src_version']
-                            and
-                            src_package['src_release'] == item['src_release']):
-                        src_packages[item['src_package']]['found'] = True
-        bootloader_uptodate = True
-        for src_package in src_packages.values():
-            if not src_package['found']:
-                bootloader_uptodate = False
-        # Bootloader built from the latest packages already in stream
-        if bootloader_uptodate:
-            print(
-                "Product %s built from the latest package set, skipping"
-                % product_id)
-            continue
-        # Find an unused version
-        today = datetime.utcnow().strftime('%Y%m%d')
-        point = 0
-        while True:
-            version = "%s.%d" % (today, point)
-            products = product_tree['products']
-            if (
-                    product_id not in products or
-                    version not in products[product_id]['versions'].keys()):
-                break
-            point += 1
-        if product_tree['products'].get(product_id) is None:
-            print("Creating new product %s" % product_id)
-            product_tree['products'][product_id] = {
-                'label': 'daily',
-                'arch': firmware_platform['arch'],
-                'arches': firmware_platform['arches'],
-                'os': firmware_platform['os'],
-                'bootloader-type': firmware_platform['firmware-platform'],
-                'versions': {},
-                }
-        path = os.path.join(
-            'bootloaders', firmware_platform['firmware-platform'],
-            firmware_platform['arch'], version)
-        dest = os.path.join(args.target, path)
-        os.makedirs(dest)
-        grub_format = firmware_platform.get('grub_format')
-        if grub_format is not None:
-            dest = os.path.join(dest, firmware_platform['grub_output'])
-        print(
-            "Downloading and creating %s version %s" % (
-                product_id, version))
-        items = extract_files_from_packages(
-            firmware_platform['archive'], firmware_platform['packages'],
-            firmware_platform['arch'], firmware_platform['files'],
-            firmware_platform['release'], args.target, path, grub_format,
-            firmware_platform.get('grub_config'),
-            firmware_platform.get('grub_output'))
-        product_tree['products'][product_id]['versions'][version] = {
-            'items': items
-        }
-
-
-def main_import(args):
-    cfg_path = os.path.join(
-        os.path.dirname(__file__), "..", "..", "conf", args.import_cfg)
-    if not os.path.exists(cfg_path):
-        if os.path.exists(args.import_cfg):
-            cfg_path = args.import_cfg
-        else:
-            print("Error: Unable to find config file %s" % args.import_cfg)
-            sys.exit(1)
-
-    with open(cfg_path) as fp:
-        cfgdata = yaml.load(fp)
-
-    target_product_stream = os.path.join(
-        'streams', 'v1', cfgdata['content_id'] + '.json')
-
-    product_tree = util.empty_iid_products(cfgdata['content_id'])
-    product_tree['products'] = load_products(
-        args.target, [target_product_stream])
-    product_tree['updated'] = sutil.timestamp()
-    product_tree['datatype'] = 'image-downloads'
-
-    if cfgdata.get('sha256_meta_data_path', None) is not None:
-        import_sha256(args, product_tree, cfgdata)
-    elif cfgdata.get('bootloaders', None) is not None:
-        import_bootloaders(args, product_tree, cfgdata)
-    else:
-        sys.stderr.write('Unsupported import yaml!')
-        sys.exit(1)
-
-    md_d = os.path.join(args.target, 'streams', 'v1')
-    if not os.path.exists(md_d):
-        os.makedirs(md_d)
-
-    with open(os.path.join(args.target, target_product_stream), 'wb') as fp:
-        fp.write(util.dump_data(product_tree))
-
-    gen_index_and_sign(args.target, not args.no_sign)
-
-
 def main_merge(args):
-    src_product_streams = load_product_streams(args.src)
-    target_product_streams = load_product_streams(args.target)
-    src_products = load_products(args.src, src_product_streams)
-    target_products = load_products(args.target, target_product_streams)
+    src_product_streams = util.load_product_streams(args.src)
+    target_product_streams = util.load_product_streams(args.target)
+    src_products = util.load_products(args.src, src_product_streams)
+    target_products = util.load_products(args.target, target_product_streams)
 
     for (product_name, product_info) in src_products.items():
         for (version, version_info) in product_info['versions'].items():
@@ -686,7 +275,7 @@
             os.path.join(args.src, product_stream),
             os.path.join(args.target, product_stream))
 
-    gen_index_and_sign(args.target, not args.no_sign)
+    util.gen_index_and_sign(args.target, not args.no_sign)
 
 
 def main_promote(args):
@@ -719,7 +308,7 @@
                                    label=args.label)
     tmirror.sync(smirror, src_path)
 
-    gen_index_and_sign(args.target, not args.no_sign)
+    util.gen_index_and_sign(args.target, not args.no_sign)
     return 0
 
 
@@ -747,7 +336,7 @@
     tmirror = BareMirrorWriter(config=mirror_config, objectstore=tstore)
     tmirror.sync(smirror, mirror_path)
 
-    gen_index_and_sign(mirror_url, not args.no_sign)
+    util.gen_index_and_sign(mirror_url, not args.no_sign)
     return 0
 
 
@@ -810,10 +399,25 @@
 
 
 def main_sign(args):
-    gen_index_and_sign(args.data_d)
+    util.gen_index_and_sign(args.data_d)
     return 0
 
 
+def main_import(args):
+    """meph2-util import wraps the preferred command 'meph2-import'.
+
+    'meph2-util import' is left for backwards compatibility, but relies
+    on modules not in the standard library in python3.2 (specifically lzma).
+    meph2-util needs to run with only dependencies available in the
+    Ubuntu 12.04 (precise) distro."""
+
+    sys.stderr.write(
+       "=== WARNING: DEPRECATED ===\n" + main_import.__doc__ + "\n")
+
+    from meph2.commands import mimport
+    return(mimport.main_import(args))
+
+
 def main():
     parser = argparse.ArgumentParser()
 

=== added file 'meph2/commands/mimport.py'
--- meph2/commands/mimport.py	1970-01-01 00:00:00 +0000
+++ meph2/commands/mimport.py	2016-09-26 19:23:59 +0000
@@ -0,0 +1,306 @@
+#!/usr/bin/python3
+
+import argparse
+from datetime import datetime
+import hashlib
+import os
+import re
+import subprocess
+import sys
+import yaml
+
+from meph2 import util
+from meph2.commands.dpkg import (
+    get_package,
+    extract_files_from_packages,
+)
+
+from meph2.commands.flags import COMMON_ARGS, SUBCOMMANDS
+from meph2.url_helper import geturl_text
+
+
+def import_sha256(args, product_tree, cfgdata):
+    for (release, release_info) in cfgdata['versions'].items():
+        if 'arch' in release_info:
+            arch = release_info['arch']
+        else:
+            arch = cfgdata['arch']
+        if 'os' in release_info:
+            os_name = release_info['os']
+        else:
+            os_name = cfgdata['os']
+        if 'path_version' in release_info:
+            path_version = release_info['path_version']
+        else:
+            path_version = release_info['version']
+        product_id = cfgdata['product_id'].format(
+            version=release_info['version'], arch=arch)
+        url = cfgdata['sha256_meta_data_path'].format(version=path_version)
+        images = get_sha256_meta_images(url)
+        base_url = os.path.dirname(url)
+
+        if product_tree['products'].get(product_id) is None:
+            print("Creating new product %s" % product_id)
+            product_tree['products'][product_id] = {
+                'subarches': 'generic',
+                'label': 'daily',
+                'subarch': 'generic',
+                'arch': arch,
+                'os': os_name,
+                'version': release_info['version'],
+                'release': release,
+                'versions': {},
+            }
+
+        for (image, image_info) in images.items():
+            if (
+                    product_id in product_tree['products'] and
+                    image in product_tree['products'][product_id]['versions']):
+                print(
+                    "Product %s at version %s exists, skipping" % (
+                        product_id, image))
+                continue
+            print(
+                "Downloading and creating %s version %s" % (
+                    (product_id, image)))
+            image_path = '/'.join([release, arch, image, 'root-tgz'])
+            real_image_path = os.path.join(
+                os.path.realpath(args.target), image_path)
+            sha256 = import_qcow2(
+                '/'.join([base_url, image_info['img_name']]),
+                image_info['sha256'], real_image_path,
+                release_info.get('curtin_files'))
+            product_tree['products'][product_id]['versions'][image] = {
+                'items': {
+                    'root-image.gz': {
+                        'ftype': 'root-tgz',
+                        'sha256': sha256,
+                        'path': image_path,
+                        'size': os.path.getsize(real_image_path),
+                        }
+                    }
+                }
+
+
+def import_bootloaders(args, product_tree, cfgdata):
+    for firmware_platform in cfgdata['bootloaders']:
+        product_id = cfgdata['product_id'].format(
+            os=firmware_platform['os'],
+            firmware_platform=firmware_platform['firmware-platform'],
+            arch=firmware_platform['arch'])
+        # Compile a list of the latest packages in the archive this bootloader
+        # pulls files from
+        src_packages = {}
+        for package in firmware_platform['packages']:
+            package_info = get_package(
+                firmware_platform['archive'], package,
+                firmware_platform['arch'], firmware_platform['release'])
+            # Some source packages include the package version in the source
+            # name. Only take the name, not the version.
+            src_package_name = package_info['Source'].split(' ')[0]
+            src_packages[src_package_name] = {
+                'src_version': package_info['Version'],
+                'src_release': firmware_platform['release'],
+                'found': False,
+            }
+        # Check if the bootloader has been built from the latest version of
+        # the packages in the archive
+        if product_id in product_tree['products']:
+            versions = product_tree['products'][product_id]['versions']
+            for data in versions.values():
+                for item in data['items'].values():
+                    src_package = src_packages.get(item['src_package'])
+                    if (
+                            src_package is not None and
+                            src_package['src_version'] == item['src_version']
+                            and
+                            src_package['src_release'] == item['src_release']):
+                        src_packages[item['src_package']]['found'] = True
+        bootloader_uptodate = True
+        for src_package in src_packages.values():
+            if not src_package['found']:
+                bootloader_uptodate = False
+        # Bootloader built from the latest packages already in stream
+        if bootloader_uptodate:
+            print(
+                "Product %s built from the latest package set, skipping"
+                % product_id)
+            continue
+        # Find an unused version
+        today = datetime.utcnow().strftime('%Y%m%d')
+        point = 0
+        while True:
+            version = "%s.%d" % (today, point)
+            products = product_tree['products']
+            if (
+                    product_id not in products or
+                    version not in products[product_id]['versions'].keys()):
+                break
+            point += 1
+        if product_tree['products'].get(product_id) is None:
+            print("Creating new product %s" % product_id)
+            product_tree['products'][product_id] = {
+                'label': 'daily',
+                'arch': firmware_platform['arch'],
+                'arches': firmware_platform['arches'],
+                'os': firmware_platform['os'],
+                'bootloader-type': firmware_platform['firmware-platform'],
+                'versions': {},
+                }
+        path = os.path.join(
+            'bootloaders', firmware_platform['firmware-platform'],
+            firmware_platform['arch'], version)
+        dest = os.path.join(args.target, path)
+        os.makedirs(dest)
+        grub_format = firmware_platform.get('grub_format')
+        if grub_format is not None:
+            dest = os.path.join(dest, firmware_platform['grub_output'])
+        print(
+            "Downloading and creating %s version %s" % (
+                product_id, version))
+        items = extract_files_from_packages(
+            firmware_platform['archive'], firmware_platform['packages'],
+            firmware_platform['arch'], firmware_platform['files'],
+            firmware_platform['release'], args.target, path, grub_format,
+            firmware_platform.get('grub_config'),
+            firmware_platform.get('grub_output'))
+        product_tree['products'][product_id]['versions'][version] = {
+            'items': items
+        }
+
+
+def get_sha256_meta_images(url):
+    """ Given a URL to a SHA256SUM file return a dictionary of filenames and
+        SHA256 checksums keyed off the file version found as a date string in
+        the filename. This is used in cases where simplestream data isn't
+        avalible.
+    """
+    ret = dict()
+    content = geturl_text(url)
+    # http://cloud.centos.org/centos/ contains images using two version
+    # strings. The first is only used on older images and uses the format
+    # YYYYMMDD_XX. The second is used on images generated monthly using the
+    # format YYMM. We know the second format is referencing the year and month
+    # by looking at the timestamp of each image.
+    prog = re.compile('([\d]{8}(_[\d]+))|(\d{4})')
+
+    for i in content.split('\n'):
+        try:
+            sha256, img_name = i.split()
+        except ValueError:
+            continue
+        if (not img_name.endswith('qcow2.xz') and
+                not img_name.endswith('qcow2')):
+            continue
+        m = prog.search(img_name)
+        if m is None:
+            continue
+        img_version = m.group(0)
+
+        # Turn the short version string into a long version string so that MAAS
+        # uses the latest version, not the longest
+        if len(img_version) == 4:
+            img_version = "20%s01_01" % img_version
+
+        # Prefer compressed image over uncompressed
+        if (img_version in ret and
+                ret[img_version]['img_name'].endswith('qcow2.xz')):
+            continue
+        ret[img_version] = {
+            'img_name': img_name,
+            'sha256': sha256,
+            }
+    return ret
+
+
+def import_qcow2(url, expected_sha256, out, curtin_files=None):
+    """ Call the maas-qcow2targz script to convert a qcow2 or qcow2.xz file at
+        a given URL or local path. Return the SHA256SUM of the outputted file.
+    """
+    # Assume maas-qcow2targz is in the path
+    qcow2targz_cmd = ["maas-qcow2targz", url, expected_sha256, out]
+    if curtin_files:
+        curtin_path = os.path.join(
+            os.path.dirname(__file__), "..", "..", "curtin")
+        qcow2targz_cmd.append(curtin_files.format(curtin_path=curtin_path))
+    proc = subprocess.Popen(qcow2targz_cmd)
+    proc.communicate()
+    if proc.wait() != 0:
+        raise subprocess.CalledProcessError(
+            cmd=qcow2targz_cmd, returncode=proc.returncode)
+
+    sha256 = hashlib.sha256()
+    with open(out, 'rb') as fp:
+        while True:
+            chunk = fp.read(2**20)
+            if not chunk:
+                break
+            sha256.update(chunk)
+    return sha256.hexdigest()
+
+
+def main_import(args):
+    cfg_path = os.path.join(
+        os.path.dirname(__file__), "..", "..", "conf", args.import_cfg)
+    if not os.path.exists(cfg_path):
+        if os.path.exists(args.import_cfg):
+            cfg_path = args.import_cfg
+        else:
+            print("Error: Unable to find config file %s" % args.import_cfg)
+            sys.exit(1)
+
+    with open(cfg_path) as fp:
+        cfgdata = yaml.load(fp)
+
+    target_product_stream = os.path.join(
+        'streams', 'v1', cfgdata['content_id'] + '.json')
+
+    product_tree = util.empty_iid_products(cfgdata['content_id'])
+    product_tree['products'] = util.load_products(
+        args.target, [target_product_stream])
+    product_tree['updated'] = util.timestamp()
+    product_tree['datatype'] = 'image-downloads'
+
+    if cfgdata.get('sha256_meta_data_path', None) is not None:
+        import_sha256(args, product_tree, cfgdata)
+    elif cfgdata.get('bootloaders', None) is not None:
+        import_bootloaders(args, product_tree, cfgdata)
+    else:
+        sys.stderr.write('Unsupported import yaml!')
+        sys.exit(1)
+
+    md_d = os.path.join(args.target, 'streams', 'v1')
+    if not os.path.exists(md_d):
+        os.makedirs(md_d)
+
+    with open(os.path.join(args.target, target_product_stream), 'wb') as fp:
+        fp.write(util.dump_data(product_tree))
+
+    util.gen_index_and_sign(args.target, not args.no_sign)
+
+
+def main():
+    subc = SUBCOMMANDS['import']
+    parser = argparse.ArgumentParser(description=subc['help'])
+
+    # Top level args
+    for (args, kwargs) in COMMON_ARGS:
+        parser.add_argument(*args, **kwargs)
+
+    for (args, kwargs) in subc['opts']:
+        if isinstance(args, str):
+            args = [args]
+        parser.add_argument(*args, **kwargs)
+    parser.set_defaults(action=main_import)
+
+    args = parser.parse_args()
+    if not getattr(args, 'action', None):
+        # http://bugs.python.org/issue16308
+        parser.print_help()
+        return 1
+
+    return args.action(args)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

=== modified file 'meph2/util.py'
--- meph2/util.py	2016-01-25 18:35:31 +0000
+++ meph2/util.py	2016-09-26 19:23:59 +0000
@@ -15,6 +15,7 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with Simplestreams.  If not, see <http://www.gnu.org/licenses/>.
 
+from simplestreams import contentsource as scontentsource
 from simplestreams import util as sutil
 from simplestreams import mirrors
 
@@ -28,6 +29,11 @@
 import sys
 import tempfile
 
+# for callers convenience
+timestamp = sutil.timestamp
+
+STREAMS_D = "streams/v1/"
+
 
 def create_index(target_d, files=None, path_prefix="streams/v1/"):
     if files is None:
@@ -253,4 +259,37 @@
     return bytestr
 
 
+def load_products(path, product_streams):
+    products = {}
+    for product_stream in product_streams:
+        product_stream_path = os.path.join(path, product_stream)
+        if os.path.exists(product_stream_path):
+            with scontentsource.UrlContentSource(
+                    product_stream_path) as tcs:
+                product_listing = sutil.load_content(tcs.read())
+                products.update(product_listing['products'])
+    return products
+
+
+def load_product_streams(src):
+    index_path = os.path.join(src, STREAMS_D, "index.json")
+    if not os.path.exists(index_path):
+        return []
+    with scontentsource.UrlContentSource(index_path) as tcs:
+        index = sutil.load_content(tcs.read())
+    return [product['path'] for product in index['index'].values()]
+
+
+def gen_index_and_sign(data_d, sign=True):
+    md_d = os.path.join(data_d, "streams", "v1")
+    if not os.path.exists(md_d):
+        os.makedirs(md_d)
+    index = create_index(md_d, files=None)
+    with open(os.path.join(md_d, "index.json"), "wb") as fp:
+        fp.write(dump_data(index))
+
+    if sign:
+        sign_streams_d(md_d)
+
+
 # vi: ts=4 expandtab syntax=python