Merge lp:~mikemc/simplestreams/add-filters-glance into lp:~smoser/simplestreams/trunk

[Mike McCracken]

- Add item and index filters to glance mirror

- Add item and index filters to glance mirror

Includes changes to sstream-mirror-glance to test item filters.

[lp:~mikemc/simplestreams/add-filters-glance]

347. By Scott Moser on 2014-06-12

Rename filter_index_item to filter_dict

=== modified file 'simplestreams/filters.py'
--- simplestreams/filters.py	2013-09-04 15:55:37 +0000
+++ simplestreams/filters.py	2014-06-12 17:30:34 +0000
@@ -64,7 +64,12 @@
 
 
 def filter_item(filters, data, src, pedigree):
-    data = util.products_exdata(src, pedigree)
+    "Apply filter list to a products entity.  Flatten before doing so."
+    return filter_dict(filters, util.products_exdata(src, pedigree))
+
+
+def filter_dict(filters, data):
+    "Apply filter list to dict. Does not flatten."
     for f in filters:
         if not f.matches(data):
             return False

=== modified file 'simplestreams/mirrors/glance.py'
--- simplestreams/mirrors/glance.py	2014-06-04 20:57:44 +0000
+++ simplestreams/mirrors/glance.py	2014-06-12 17:30:34 +0000
@@ -15,6 +15,7 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with Simplestreams.  If not, see <http://www.gnu.org/licenses/>.
 
+import simplestreams.filters as filters
 import simplestreams.mirrors as mirrors
 import simplestreams.util as util
 import simplestreams.openstack as openstack
@@ -44,6 +45,17 @@
                  name_prefix=None):
         super(GlanceMirror, self).__init__(config=config)
 
+        self.item_filters = self.config.get('item_filters', [])
+        if len(self.item_filters) == 0:
+            self.item_filters = ['ftype~(disk1.img|disk.img)',
+                                 'arch~(x86_64|amd64|i386)']
+        self.item_filters = filters.get_filters(self.item_filters)
+
+        self.index_filters = self.config.get('index_filters', [])
+        if len(self.index_filters) == 0:
+            self.index_filters = ['datatype=image-downloads']
+        self.index_filters = filters.get_filters(self.index_filters)
+
         self.loaded_content = {}
         self.store = objectstore
 
@@ -134,9 +146,7 @@
         return glance_t
 
     def filter_item(self, data, src, target, pedigree):
-        flat = util.products_exdata(src, pedigree, include_top=False)
-        return (flat.get('ftype') in ('disk1.img', 'disk.img') and
-                flat.get('arch') in ('x86_64', 'amd64', 'i386'))
+        return filters.filter_item(self.item_filters, data, src, pedigree)
 
     def insert_item(self, data, src, target, pedigree, contentsource):
         flat = util.products_exdata(src, pedigree, include_top=False)
@@ -215,7 +225,7 @@
             self.gclient.images.delete(data['id'])
 
     def filter_index_entry(self, data, src, pedigree):
-        return data.get('datatype') in ("image-downloads", None)
+        return filters.filter_dict(self.index_filters, data)
 
     def insert_products(self, path, target, content):
         if not self.store:

=== modified file 'tools/sstream-mirror-glance'
--- tools/sstream-mirror-glance	2013-09-09 19:45:15 +0000
+++ tools/sstream-mirror-glance	2014-06-12 17:30:34 +0000
@@ -20,6 +20,7 @@
 # glanceclient) are not python3.
 #
 import argparse
+import logging
 import os.path
 import sys
 
@@ -79,6 +80,14 @@
 
     parser.add_argument('source_mirror')
     parser.add_argument('path', nargs='?', default="streams/v1/index.sjson")
+    parser.add_argument('--item_filter', action='append', default=[],
+                        dest="item_filters",
+                        help="Filter expression for mirrored items. "
+                        "Multiple filter arguments can be specified"
+                        "and will be combined with logical AND. "
+                        "Expressions are key[!]=literal_string "
+                        "or key[!]~regexp.")
+
     args = parser.parse_args()
 
     modify_hook = None
@@ -87,7 +96,8 @@
 
     mirror_config = {'max_items': args.max, 'keep_items': args.keep,
                      'cloud_name': args.cloud_name,
-                     'modify_hook': modify_hook}
+                     'modify_hook': modify_hook,
+                     'item_filters': args.item_filters}
 
     def policy(content, path):  # pylint: disable=W0613
         if args.path.endswith('sjson'):