Merge lp:~jamestait/click-reviewers-tools/enable-store-metadata-report into lp:click-reviewers-tools

[James Tait]

Add support for extracting package metadata for the Store.

This branch adds a new script that the Store can use to extract metadata from packages and have it returned in a consistent format.

Unmerged revisions

617. By James Tait on 2016-04-06

Add support for extracting package metadata for the Store.

=== added file 'bin/click-metadata'
--- bin/click-metadata	1970-01-01 00:00:00 +0000
+++ bin/click-metadata	2016-04-06 09:31:46 +0000
@@ -0,0 +1,27 @@
+#!/usr/bin/python3
+
+import json
+import sys
+
+from clickreviews import (
+    common,
+    cr_common,
+    sr_common,
+)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        common.error("Must give path to package")
+
+    review = common.Review(sys.argv[1], '')
+
+    if review.is_click or review.is_snap1:
+        metadata = cr_common.ClickReview(sys.argv[1], 'metadata')
+    elif review.is_snap2:
+        metadata = sr_common.SnapReview(sys.argv[1], 'metadata')
+    else:
+        common.error('Unknown package type.')
+
+    print(json.dumps(metadata.get_metadata(), sort_keys=True,
+                     indent=2, separators=(',', ': ')))

=== modified file 'clickreviews/common.py'
--- clickreviews/common.py	2016-02-23 21:09:07 +0000
+++ clickreviews/common.py	2016-04-06 09:31:46 +0000
@@ -373,6 +373,62 @@
         '''Set review name'''
         self.review_type = name
 
+    #
+    # Package Metadata extraction, primarily for the Store.
+    #
+
+    def get_metadata(self):
+        metadata = {
+            'name': self.get_name(),
+            'version': self.get_version(),
+            'title': self.get_title(),
+            'description': self.get_description(),
+            'architecture': self.pkg_arch,
+        }
+        framework = self.get_framework()
+        if framework is not None:
+            metadata['framework'] = framework
+        icon = self.get_icon()
+        if icon['path'] is not None:
+            metadata['icon'] = icon
+        installed_size = self.get_installed_size()
+        if installed_size is not None:
+            metadata['installed-size'] = installed_size
+        maintainer = self.get_maintainer()
+        if maintainer is not None:
+            metadata['maintainer'] = maintainer
+        return [
+            metadata, {
+                'is_snap': self.is_snap1 or self.is_snap2,
+                'is_squashfs': self.is_snap2
+            }]
+
+    def get_name(self):
+        return None
+
+    def get_version(self):
+        return None
+
+    def get_installed_size(self):
+        return None
+
+    def get_title(self):
+        return None
+
+    def get_description(self):
+        return None
+
+    def get_framework(self):
+        return None
+
+    def get_icon(self):
+        return None
+
+    def get_architecture(self):
+        return None
+
+    def get_maintainer(self):
+        return None
 
 #
 # Utility functions
@@ -541,6 +597,16 @@
     return dest
 
 
+def open_binary_file_read(path):
+    '''Open specified binary file read-only'''
+    try:
+        orig = codecs.open(path, 'rb')
+    except Exception:
+        raise
+
+    return orig
+
+
 def create_tempdir():
     '''Create/reuse a temporary directory that is automatically cleaned up'''
     global TMP_DIR

=== modified file 'clickreviews/cr_common.py'
--- clickreviews/cr_common.py	2016-02-09 22:49:57 +0000
+++ clickreviews/cr_common.py	2016-04-06 09:31:46 +0000
@@ -16,6 +16,7 @@
 
 from __future__ import print_function
 from debian.deb822 import Deb822
+import base64
 import glob
 import json
 import os
@@ -28,6 +29,7 @@
     Review,
     ReviewException,
     error,
+    open_binary_file_read,
     open_file_read,
 )
 
@@ -391,6 +393,43 @@
                 d[name][key] = entry[key]
         return d
 
+    def get_name(self):
+        return self.click_pkgname
+
+    def get_version(self):
+        return self.click_version
+
+    def get_installed_size(self):
+        return self.manifest.get('installed-size')
+
+    def get_title(self):
+        return self.manifest.get('title')
+
+    def get_description(self):
+        return self.manifest.get('description')
+
+    def get_framework(self):
+        return self.manifest.get('framework')
+
+    def get_icon(self):
+        icon_path = self.manifest.get('icon')
+        icon_data = None
+        if icon_path:
+            full_path = os.path.join(self.unpack_dir, icon_path)
+            with open_binary_file_read(full_path) as f:
+                icon_data = base64.b64encode(f.read())
+        return {
+            'path': icon_path,
+            'data': str(icon_data) if icon_data else None,
+            'filename': os.path.basename(icon_path) if icon_path else None,
+        }
+
+    def get_architecture(self):
+        return self.manifest.get('architecture')
+
+    def get_maintainer(self):
+        return self.manifest.get('maintainer')
+
     def check_peer_hooks(self, hooks_sublist=[]):
         '''Check if peer hooks are valid'''
         # Nothing to verify

=== added file 'clickreviews/cr_metadata.py'
--- clickreviews/cr_metadata.py	1970-01-01 00:00:00 +0000
+++ clickreviews/cr_metadata.py	2016-04-06 09:31:46 +0000
@@ -0,0 +1,106 @@
+'''cr_content_hub.py: click content-hub checks'''
+#
+# Copyright (C) 2014-2015 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import print_function
+
+from clickreviews.cr_common import ClickReview, error, open_file_read
+import json
+import os
+
+
+class ClickReviewMetadata(ClickReview):
+    '''This class represents click lint reviews'''
+    def __init__(self, fn, overrides=None):
+        ClickReview.__init__(self, fn, "metadata", overrides=overrides)
+        if not self.is_click and not self.is_snap1:
+            return
+
+        self.valid_keys = ['destination', 'share', 'source']
+
+        self.content_hub_files = dict()  # click-show-files and tests
+        self.content_hub = dict()
+
+        if self.manifest is None:
+            return
+
+        for app in self.manifest['hooks']:
+            if 'content-hub' not in self.manifest['hooks'][app]:
+                # msg("Skipped missing content-hub hook for '%s'" % app)
+                continue
+            if not isinstance(self.manifest['hooks'][app]['content-hub'], str):
+                error("manifest malformed: hooks/%s/urls is not str" % app)
+            (full_fn, jd) = self._extract_content_hub(app)
+            self.content_hub_files[app] = full_fn
+            self.content_hub[app] = jd
+
+    def _extract_content_hub(self, app):
+        '''Get content-hub hook content'''
+        c = self.manifest['hooks'][app]['content-hub']
+        fn = os.path.join(self.unpack_dir, c)
+
+        bn = os.path.basename(fn)
+        if not os.path.exists(fn):
+            error("Could not find '%s'" % bn)
+
+        fh = open_file_read(fn)
+        contents = ""
+        for line in fh.readlines():
+                contents += line
+        fh.close()
+
+        try:
+            jd = json.loads(contents)
+        except Exception as e:
+            error("content-hub json unparseable: %s (%s):\n%s" % (bn,
+                  str(e), contents))
+
+        if not isinstance(jd, dict):
+            error("content-hub json is malformed: %s:\n%s" % (bn, contents))
+
+        return (fn, jd)
+
+    def check_valid(self):
+        '''Check validity of content-hub entries'''
+        if not self.is_click and not self.is_snap1:
+            return
+
+        for app in sorted(self.content_hub):
+            for k in self.content_hub[app].keys():
+                t = "info"
+                n = self._get_check_name('valid', app=app, extra=k)
+                s = "OK"
+
+                if not isinstance(self.content_hub[app][k], list):
+                    t = "error"
+                    s = "'%s' is not a list" % k
+                elif len(self.content_hub[app][k]) < 1:
+                    t = "error"
+                    s = "'%s' is empty" % k
+                self._add_result(t, n, s)
+                if t == "error":
+                    continue
+
+                for v in self.content_hub[app][k]:
+                    t = "info"
+                    n = self._get_check_name('valid_value', app=app, extra=k)
+                    s = "OK"
+                    if not isinstance(v, str):
+                        t = "error"
+                        s = "'%s' is not a string" % k
+                    elif v == "":
+                        t = "error"
+                        s = "'%s' is empty" % k
+                    self._add_result(t, n, s)

=== modified file 'clickreviews/sr_common.py'
--- clickreviews/sr_common.py	2016-03-14 19:20:55 +0000
+++ clickreviews/sr_common.py	2016-04-06 09:31:46 +0000
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import print_function
+import base64
 import os
 import re
 import yaml
@@ -24,6 +25,7 @@
     Review,
     ReviewException,
     error,
+    open_binary_file_read,
     open_file_read,
 )
 
@@ -129,3 +131,34 @@
         if pat.search(n):
             return True
         return False
+
+    def get_name(self):
+        return self.snap_yaml.get('name')
+
+    def get_version(self):
+        return str(self.snap_yaml.get('version'))
+
+    def get_description(self):
+        return self.snap_yaml.get('description')
+
+    def get_icon(self):
+        icon_path = None
+        for ext in ('svg', 'png'):
+            path = 'meta/icon.' + ext
+            full_path = os.path.join(self.unpack_dir, path)
+            if os.path.isfile(full_path):
+                icon_path = path
+                break
+        icon_data = None
+        if icon_path:
+            full_path = os.path.join(self.unpack_dir, icon_path)
+            with open_binary_file_read(full_path) as f:
+                icon_data = base64.b64encode(f.read())
+        return {
+            'path': icon_path,
+            'data': str(icon_data) if icon_data else None,
+            'filename': os.path.basename(icon_path) if icon_path else None,
+        }
+
+    def get_architecture(self):
+        return self.pkg_arch