Merge lp:~ltrager/maas-images/create into lp:maas-images

[Lee Trager]

Add create command which allows creation of streams for arbitrary images.

The create command allows creating streams for arbitrary images. It can be used for ubuntu-core as follows

wget http://cdimage.ubuntu.com/ubuntu-core/16/stable/current/ubuntu-core-16-amd64.img.xz

meph2-util create --fields arch=amd64,kernel_snap=pc-kernel,gadget_snap=pc,gadget_title='PC',model_name=pc-amd64,brand_id=canonical,channel=stable,label=daily,os=ubuntu-core,os_title='Ubuntu Core',release=16,release_title='16',maas_supported=2.2,support_eol=2018-04-26 --image-path-postfix 16/pc ubuntu-core amd64 com.ubuntu.maas.daily:v4:download com.ubuntu.maas.daily:v4:16:amd64:pc:stable root-dd.xz curtin-ubuntu-core-16-amd64.img.xz ubuntu-core

Sample output - http://162.213.35.187/ubuntu-core/

[lp:~ltrager/maas-images/create]

360. By Lee Trager on 2017-03-29

Set the item name to the ftype

361. By Lee Trager on 2017-03-29

Set the filename to the same as the ftype

[Scott Moser (smoser)]

I think this is more generically already present in streams in json2streams.
Do you think that could be used?

[Scott Moser (smoser)]

I'm going to mark this 'rejected'.

Lee, you can feel free to re-submit it to git.

Just trying to clean things up.

Unmerged revisions

361. By Lee Trager on 2017-03-29

Set the filename to the same as the ftype

360. By Lee Trager on 2017-03-29

Set the item name to the ftype

359. By Lee Trager on 2017-03-28

Add create command which creates streams for custom images.

=== modified file 'meph2/commands/flags.py'
--- meph2/commands/flags.py	2016-09-26 15:21:43 +0000
+++ meph2/commands/flags.py	2017-03-29 19:56:36 +0000
@@ -102,6 +102,25 @@
             COMMON_FLAGS['data_d'], COMMON_FLAGS['no-sign'],
         ],
     },
+    'create': {
+        'help': 'Create a new image entry in the stream.',
+        'opts': [
+            COMMON_FLAGS['no-sign'],
+            ('--fields', {
+                'help': (
+                    'Comma seperated key value pairs to be added to listing.'),
+                }),
+            ('--image-path-postfix', {
+                'help': 'Optional path to postfix to standard image-path.'}),
+            ('os', {'help': 'The OS name'}),
+            ('arch', {'help': 'The architecture for the image.'}),
+            ('content_id', {'help': 'The content_id of the stream.'}),
+            ('product_name', {'help': 'The product name in the stream.'}),
+            ('ftype', {'help': 'The filetype of the image.'}),
+            ('image', {'help': 'The image to be imported.'}),
+            COMMON_FLAGS['target'],
+            ]
+    },
 }
 
 # vi: ts=4 expandtab syntax=python
 
=== modified file 'meph2/commands/meph2_util.py'
--- meph2/commands/meph2_util.py	2017-02-11 01:29:31 +0000
+++ meph2/commands/meph2_util.py	2017-03-29 19:56:36 +0000
@@ -1,9 +1,11 @@
 #!/usr/bin/python3
 
+from datetime import datetime
+from functools import partial
 import argparse
 import copy
+import hashlib
 import os
-from functools import partial
 import shutil
 import sys
 
@@ -13,8 +15,9 @@
 from simplestreams import (
     filters,
     mirrors,
+    objectstores,
     util as sutil,
-    objectstores)
+)
 
 
 class BareMirrorWriter(mirrors.ObjectFilterMirror):
@@ -418,6 +421,72 @@
     return(mimport.main_import(args))
 
 
+def main_create(args):
+    product_stream_path = os.path.join(
+        'streams', 'v1', args.content_id + '.json')
+    product_tree = util.empty_iid_products(args.content_id)
+    product_tree['products'] = util.load_products(
+        args.target, [product_stream_path])
+    product_tree['updated'] = util.timestamp()
+    product_tree['datatype'] = 'image-downloads'
+
+    product = product_tree['products'].get(args.product_name, {})
+    if 'versions' not in product:
+        product['os'] = args.os
+        product['arch'] = args.arch
+        product['versions'] = {}
+    if args.fields is not None:
+        for pair in args.fields.split(','):
+            key, value = pair.split('=')
+            product[key] = value
+
+    today = datetime.utcnow().strftime('%Y%m%d')
+    ver_str = today
+    i = 0
+    while ver_str in product['versions']:
+        i += 1
+        ver_str = '%s.%d' % (today, i)
+
+    image_path = os.path.join(args.os, args.arch, ver_str)
+    if args.image_path_postfix is not None:
+        image_path = os.path.join(image_path, args.image_path_postfix)
+    image_path = os.path.join(image_path, args.ftype)
+    full_image_path = os.path.join(args.target, image_path)
+    os.makedirs(os.path.dirname(full_image_path))
+    shutil.copy2(args.image, full_image_path)
+
+    sha256 = hashlib.sha256()
+    size = 0
+    with open(args.image, 'rb') as fp:
+        while True:
+            chunk = fp.read(2**20)
+            if not chunk:
+                break
+            sha256.update(chunk)
+            size += len(chunk)
+
+    product['versions'][ver_str] = {
+        'items': {
+            args.ftype: {
+                'ftype': args.ftype,
+                'path': image_path,
+                'sha256': sha256.hexdigest(),
+                'size': size,
+            },
+        },
+    }
+    product_tree['products'][args.product_name] = product
+
+    full_product_stream_path = os.path.join(args.target, product_stream_path)
+    full_product_stream_dir = os.path.dirname(full_product_stream_path)
+    if not os.path.exists(full_product_stream_dir):
+        os.makedirs(full_product_stream_dir)
+    with open(full_product_stream_path, 'wb') as fp:
+        fp.write(util.dump_data(product_tree))
+
+    util.gen_index_and_sign(args.target, not args.no_sign)
+
+
 def main():
     parser = argparse.ArgumentParser()