Merge ~oddbloke/simplestreams/+git/simplestreams:ubuntu/devel into simplestreams:ubuntu/devel

[Dan Watkins]

releasing simplestreams version 0.1.0-25-gba75825b-0ubuntu1

[Server Team CI bot (server-team-bot)]

No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want jenkins to rebuild you need to trigger it yourself):
https://code.launchpad.net/~daniel-thewatkins/simplestreams/+git/simplestreams/+merge/371523/+edit-commit-message
https://jenkins.ubuntu.com/server/job/simplestreams-ci/87/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/simplestreams-ci/nodes=metal-amd64/87/
    SUCCESS: https://jenkins.ubuntu.com/server/job/simplestreams-ci/nodes=metal-ppc64el/87/
    SUCCESS: https://jenkins.ubuntu.com/server/job/simplestreams-ci/nodes=metal-s390x/87/

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/simplestreams-ci/87//rebuild

review: Needs Fixing (continuous-integration)

[Server Team CI bot (server-team-bot)]

No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want jenkins to rebuild you need to trigger it yourself):
https://code.launchpad.net/~daniel-thewatkins/simplestreams/+git/simplestreams/+merge/371523/+edit-commit-message
https://jenkins.ubuntu.com/server/job/simplestreams-ci/88/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/simplestreams-ci/nodes=metal-amd64/88/
    SUCCESS: https://jenkins.ubuntu.com/server/job/simplestreams-ci/nodes=metal-ppc64el/88/
    SUCCESS: https://jenkins.ubuntu.com/server/job/simplestreams-ci/nodes=metal-s390x/88/

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/simplestreams-ci/88//rebuild

review: Needs Fixing (continuous-integration)

[Robert C Jennings (rcj)]

+1 on the changes

review: Approve

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:94a8cd6f9425f37e3b9e5c91ff22d00926630a5b
https://jenkins.ubuntu.com/server/job/simplestreams-ci/89/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/simplestreams-ci/nodes=metal-amd64/89/
    SUCCESS: https://jenkins.ubuntu.com/server/job/simplestreams-ci/nodes=metal-ppc64el/89/
    SUCCESS: https://jenkins.ubuntu.com/server/job/simplestreams-ci/nodes=metal-s390x/89/

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/simplestreams-ci/89//rebuild

review: Approve (continuous-integration)

diff --git a/debian/changelog b/debian/changelog
index 27e4398..41e1af5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,13 @@
+simplestreams (0.1.0-25-gba75825b-0ubuntu1) eoan; urgency=medium
+
+  * New upstream snapshot.
+    - Add combined_disk1-img_sha256 to download.json [Stéphane Graber]
+    - Apply filters to items before filtering versions [Philip Roche]
+    - Add KVM images to eoan. [Pat Viafore]
+  * d/control: remove python-simplestreams-openstack
+
+ -- Daniel Watkins <[e-mail]>  Tue, 20 Aug 2019 10:31:20 -0400
+
 simplestreams (0.1.0-22-g409fdc15-0ubuntu1) disco; urgency=medium
 
   * debian/README.source: update to refer to upstream packaging
diff --git a/debian/control b/debian/control
index 6cd458d..350b2d0 100644
--- a/debian/control
+++ b/debian/control
@@ -55,19 +55,6 @@ Description: Library and tools for using Simple Streams data
  This package provides a client for interacting with simple
  streams data as is produced to describe Ubuntu's cloud images.
 
-Package: python-simplestreams-openstack
-Architecture: all
-Priority: optional
-Depends: python-glanceclient,
-         python-keystoneclient,
-         python-simplestreams,
-         python-swiftclient,
-         ${misc:Depends}
-Description: Library and tools for using Simple Streams data
- This package depends on libraries necessary to use the openstack dependent
- functionality in simplestreams.  That includes interacting with glance,
- swift and keystone.
-
 Package: python3-simplestreams-openstack
 Architecture: all
 Priority: optional
diff --git a/simplestreams/mirrors/__init__.py b/simplestreams/mirrors/__init__.py
index 2a1dc7e..60d5ade 100644
--- a/simplestreams/mirrors/__init__.py
+++ b/simplestreams/mirrors/__init__.py
@@ -280,6 +280,23 @@ class BasicMirrorWriter(MirrorWriter):
 
         filtered_products = []
         prodname = None
+
+        # Apply filters to items before filtering versions
+        for prodname, product in list(stree.items()):
+
+            for vername, version in list(product.get('versions', {}).items()):
+                for itemname, item in list(version.get('items', {}).items()):
+                    pgree = (prodname, vername, itemname)
+                    if not self.filter_item(item, src, target, pgree):
+                        LOG.debug("Filtered out item: %s/%s", itemname, item)
+                        del stree[prodname]['versions'][vername]['items'][
+                            itemname]
+                        if not stree[prodname]['versions'][vername].get(
+                                'items', {}):
+                            del stree[prodname]['versions'][vername]
+                        if not stree[prodname].get('versions', {}):
+                            del stree[prodname]
+
         for prodname, product in stree.items():
             if not self.filter_product(product, src, target, (prodname,)):
                 filtered_products.append(prodname)
@@ -320,9 +337,6 @@ class BasicMirrorWriter(MirrorWriter):
                 added_items = []
                 for itemname, item in version.get('items', {}).items():
                     pgree = (prodname, vername, itemname)
-                    if not self.filter_item(item, src, target, pgree):
-                        LOG.debug("Filtered out item: %s/%s", itemname, item)
-                        continue
 
                     added_items.append(itemname)
 
diff --git a/tools/make-test-data b/tools/make-test-data
index fc0fd1d..a1e183d 100755
--- a/tools/make-test-data
+++ b/tools/make-test-data
@@ -58,6 +58,7 @@ FAKE_DATA = {
     'lxd.tar.xz': {'size': 10245},
     'tar.gz': {'size': 11264},
     'disk1.img': {'size': 12288},
+    'disk-kvm.img': {'size': 12289},
     'uefi1.img': {'size': 12421},
     'manifest': {'size': 10241},
     'ova': {'size': 12399},
@@ -296,8 +297,10 @@ def create_fake_file(prefix, item):
     # create a combined sha256 for lxd.tar.xz (metadata) and the root fs
     # - combined_sha256 and combined_rootxz_sha256 for the -root.tar.gz
     # - combined_squashfs_sha256 for the squashfs
+    # - combined_disk1-img_sha256 for the img
     ftype = item['ftype']
-    for name, extension in (('rootxz', '-root.tar.xz'),
+    for name, extension in (('disk1-img', '.img'),
+                            ('rootxz', '-root.tar.xz'),
                             ('squashfs', '.squashfs')):
         if (ftype == "lxd.tar.xz" and
                 'combined_{}_sha256'.format(name) not in FAKE_DATA[ftype]):
@@ -566,7 +569,8 @@ def create_image_data(query_tree, out_d, streamdir):
             lxd = os.path.basename(path)
 
             # find calculated combined checksums
-            for name, extension in (('rootxz', '-root.tar.xz'),
+            for name, extension in (('disk1-img', '.img'),
+                                    ('rootxz', '-root.tar.xz'),
                                     ('squashfs', '.squashfs')):
                 root = lxd.replace('-lxd.tar.xz', extension)
                 combined = os.path.join(dirname, ','.join([lxd, root]))
diff --git a/tools/toolutil.py b/tools/toolutil.py
index 0764770..eb91e53 100644
--- a/tools/toolutil.py
+++ b/tools/toolutil.py
@@ -55,6 +55,13 @@ def is_expected(suffix, fields):
             # precise got -root.tar.gz after alpha2
             return False
 
+    if suffix == "-disk-kvm.img":
+        # kvm images were introduced into eoan
+        if codename_cmp(rel, "<", "eoan") or serial < "20190622":
+            return False
+        if arch not in ["amd64"]:
+            return False
+
     if suffix == "-disk1.img":
         # disk1.img as a suffix was replaced by .img in yakkety
         if codename_cmp(rel, "<", "oneiric"):
@@ -155,7 +162,8 @@ def load_query_download(path, builds=None, rels=None):
 
     suffixes = (".tar.gz", "-root.tar.gz", "-disk1.img", "-uefi1.img",
                 ".manifest", ".ova", "-root.tar.xz", "-root.manifest",
-                "-lxd.tar.xz", ".squashfs", ".squashfs.manifest", ".img",)
+                "-lxd.tar.xz", ".squashfs", ".squashfs.manifest", ".img",
+                "-disk-kvm.img",)
     streams = [f[0:-len(".latest.txt")]
                for f in os.listdir(path) if f.endswith("latest.txt")]