Merge lp:~blake-rouse/maas-images/clean-unsupported into lp:maas-images

[Blake Rouse]

Add clean-unsupported command to meph2-utils.

Removes old Ubuntu releases from the stream and optionally creates a stream for the items it has removed.

Can be used to remove old releases from a stream. In this case to remove old releases from the releases stream and add to an old-releases stream.

meph2-util clean-unsupported releases/ --removed-stream releases-removed/
meph2-util merge releases-removed/ old-releases/
rm -rf releases-removed

For daily we should not keep any copies so its simply:

meph2-util clean-unsupported daily/

[Lee Trager (ltrager)]

LGTM!

review: Approve

=== modified file 'meph2/commands/meph2_util.py'
--- meph2/commands/meph2_util.py	2016-04-18 22:40:10 +0000
+++ meph2/commands/meph2_util.py	2016-06-02 20:45:30 +0000
@@ -11,7 +11,7 @@
 import subprocess
 import yaml
 
-from meph2 import util
+from meph2 import util, ubuntu_info
 from meph2.url_helper import geturl_text
 
 from simplestreams import (
@@ -102,6 +102,16 @@
             ('filters', {'nargs': '*', 'default': []}),
         ]
     },
+    'clean-unsupported': {
+        'help': 'clean streams metadata only to keep supported items',
+        'opts': [
+            COMMON_FLAGS['dry-run'], COMMON_FLAGS['no-sign'],
+            COMMON_FLAGS['keyring'],
+            ('target', {}),
+            ('--removed-stream',
+             {'help': 'create stream of removed products'}),
+        ]
+    },
     'find-orphans': {
         'help': 'find files in data_d not referenced in a "path"',
         'opts': [
@@ -324,6 +334,61 @@
         self.removed_versions.append((self.tcontent_id, pedigree,))
 
 
+class UbuntuUnsupportedMirrorWriter(BareMirrorWriter):
+
+    def filter_product(self, data, src, target, pedigree):
+        data = sutil.products_exdata(src, pedigree)
+        is_ubuntu_os = (
+            'os' not in data or data['os'] == 'ubuntu')
+        if is_ubuntu_os and data['release'] not in ubuntu_info.SUPPORTED:
+            return True
+        return False
+
+
+class UbuntuCleanUnsupportedMirrorWriter(BareMirrorWriter):
+
+    def __init__(self, config, objectstore, remove):
+        super(UbuntuCleanUnsupportedMirrorWriter, self).__init__(
+            config, objectstore)
+        self.remove = remove
+
+    def sync_products(self, reader, path=None, src=None, content=None):
+        (src, content) = mirrors._get_data_content(path, src, content, reader)
+
+        sutil.expand_tree(src)
+
+        mirrors.check_tree_paths(src)
+
+        content_id = src['content_id']
+        target = self.load_products(path, content_id)
+        if not target:
+            target = sutil.stringitems(src)
+
+        sutil.expand_tree(target)
+
+        stree = src.get('products', {})
+        if 'products' not in target:
+            target['products'] = {}
+
+        for prodname, product in stree.items():
+            data = sutil.products_exdata(src, (prodname,))
+            is_ubuntu_os = (
+                'os' not in data or data['os'] == 'ubuntu')
+            if is_ubuntu_os and data['release'] not in ubuntu_info.SUPPORTED:
+                self.removed_versions.append((prodname,))
+                if self.remove:
+                    self.remove_product(product, src, target, (prodname,))
+
+        self.insert_products(path, target, content)
+
+    def remove_product(self, data, src, target, pedigree):
+        # Delete all items in all versions.
+        for vername, version in data.get('versions', {}).items():
+            for itemname, item in version.get('items', {}).items():
+                if 'path' in item:
+                    self.store.remove(item['path'])
+
+
 def get_sha256_meta_images(url):
     """ Given a URL to a SHA256SUM file return a dictionary of filenames and
         SHA256 checksums keyed off the file version found as a date string in
@@ -635,6 +700,52 @@
     return 0
 
 
+def main_clean_unsupported(args):
+    (target_url, target_path) = sutil.path_from_mirror_url(args.target, None)
+    policy = partial(util.endswith_policy, target_path, args.keyring)
+
+    if args.dry_run:
+        smirror = mirrors.UrlMirrorReader(target_url, policy=policy)
+        tstore = objectstores.FileStore(target_url)
+        tmirror = UbuntuCleanUnsupportedMirrorWriter(
+            config={}, objectstore=tstore, remove=False)
+        tmirror.sync(smirror, target_path)
+        for pedigree in tmirror.removed_versions:
+            sys.stderr.write("remove " + '/'.join(pedigree) + "\n")
+        return 0
+
+    if args.removed_stream:
+        # Copy unsupported releases to removed.
+        (removed_url, removed_path) = sutil.path_from_mirror_url(
+            args.removed_stream, None)
+        smirror = mirrors.UrlMirrorReader(target_url, policy=policy)
+        tstore = objectstores.FileStore(removed_url)
+        tmirror = UbuntuUnsupportedMirrorWriter(
+            config={'max_items': 1}, objectstore=tstore)
+        tmirror.sync(smirror, target_path)
+        gen_index_and_sign(removed_url, not args.no_sign)
+
+    # Remove the unsupported releases from target.
+    smirror = mirrors.UrlMirrorReader(target_url, policy=policy)
+    tstore = objectstores.FileStore(target_url)
+    tmirror = UbuntuCleanUnsupportedMirrorWriter(
+        config={}, objectstore=tstore, remove=True)
+    tmirror.sync(smirror, target_path)
+    gen_index_and_sign(target_url, not args.no_sign)
+
+    # Remove any directories that are now empty.
+    for dir, _, _ in os.walk(target_url, topdown=False):
+        if dir == target_url:
+            break
+        else:
+            try:
+                os.rmdir(dir)
+            except OSError:
+                pass
+
+    return 0
+
+
 def main_find_orphans(args):
     data_d = args.data_d
     streams_d = args.streams_dirs

=== modified file 'tox.ini'
--- tox.ini	2016-02-25 18:25:34 +0000
+++ tox.ini	2016-06-02 20:45:30 +0000
@@ -15,6 +15,9 @@
 [testenv:jerff]
 # the 'jerff' environment simulates what is on the build system
 # it has very few dependencies, but must be able to run meph2-util
+deps =
+    {[testenv]deps}
+    distro_info
 commands = {toxinidir}/bin/meph2-util --help
 
 [testenv:flake8]