Merge lp:~larryprice/libertine-scope/libertine-store-internal-packages into lp:~larryprice/libertine-scope/libertine-store-search

[Larry Price]

Search internal container caches for appstream or aptcache data.

Search internal container caches for appstream or aptcache data.

When creating the 'palatine' container, 'appstream' will be installed by default such that we can use it's internal cache. In order to use the container cache currently, you'll need to restart the libertined service after creating the container. However, there is also an MP which refactors container creation that will resolve this.

Unmerged revisions

112. By Larry Price on 2016-10-06

Removing raise

111. By Larry Price on 2016-10-06

Wrong endpoint for update

110. By Larry Price on 2016-10-06

Install appstream by default in any containers created by the service

109. By Larry Price on 2016-10-06

Cache classes attempt to use internal apt caches

=== modified file 'service/libertine_service/appstream.py'
--- service/libertine_service/appstream.py	2016-09-02 15:27:28 +0000
+++ service/libertine_service/appstream.py	2016-10-06 18:10:43 +0000
@@ -16,6 +16,13 @@
 gi.require_version('AppStream', '1.0')
 from gi.repository import AppStream
 
+gi.require_version('Libertine', '1')
+from gi.repository import Libertine
+
+from os import listdir
+from os.path import isfile, join
+from libertine import utils
+
 
 def xstr(s):
 	"""Converts anything to a string"""
@@ -24,15 +31,18 @@
 
 class AppStreamCache(object):
 	"""AppStreamCache"""
-	def __init__(self, config):
+	def __init__(self, container_id):
 		super(AppStreamCache, self).__init__()
-		self.config = config
-		self.pool = AppStream.Pool()
-		self.pool.load()
+		self._container = container_id
+		self._pool = AppStream.Pool()
+		self._loaded = False
+		self._basepath = ""
 
 	def search(self, query):
+		self._load()
+
 		packages = []
-		for component in self.pool.search(query):
+		for component in self._pool.search(query):
 			if component.is_valid():
 				package = {}
 				package["name"] = xstr(component.get_name())
@@ -41,13 +51,16 @@
 				package["uri"] = xstr(component.get_url(AppStream.UrlKind.HOMEPAGE))
 
 				if len(component.get_icons()) > 0:
-					package["icon"] = xstr(self._get_largest_icon(component.get_icons()).get_filename())
+					package["icon"] = join(self._basepath,
+					                       xstr(self._get_largest_icon(component.get_icons()).get_filename()))
 				packages.append(package)
 
 		return packages[::-1]
 
 	def app_info(self, app_id):
-		apps = self.pool.get_components_by_id(app_id)
+		self._load()
+
+		apps = self._pool.get_components_by_id(app_id)
 		app_data = {}
 		if len(apps) == 1 and apps[0].is_valid():
 			app = apps[0]
@@ -63,12 +76,13 @@
 					if largest is not None:
 						if "screenshots" not in app_data:
 							app_data["screenshots"] = []
-						app_data["screenshots"].append(largest.get_url())
+						app_data["screenshots"].append(join(self._basepath, largest.get_url()))
 
 			app_data["description"] = xstr(app.get_description())
 
 			if len(app.get_icons()) > 0:
-				app_data["icon"] = xstr(self._get_largest_icon(app.get_icons()).get_filename())
+				app_data["icon"] = join(self._basepath,
+				                        xstr(self._get_largest_icon(app.get_icons()).get_filename()))
 
 		return app_data
 
@@ -78,3 +92,16 @@
 			if largest is None or icon.get_width() > largest.get_width():
 				largest = icon
 		return largest
+
+	def _load(self):
+		if not self._loaded:
+			path = join(Libertine.container_path(self._container), "var/cache/app-info/gv/")
+			cachefiles = [f for f in listdir(path) if isfile(join(path, f))]
+			if len(cachefiles) > 0:
+				utils.get_logger().debug("Using appstream cache for container %s" % self._container)
+				self._pool.load_cache_file(join(path, cachefiles[0]))
+				self._basepath = path
+			else:
+				utils.get_logger().debug("Using system appstream cache")
+				self._pool.load()
+			self._loaded = True

=== modified file 'service/libertine_service/apt.py'
--- service/libertine_service/apt.py	2016-09-20 17:56:18 +0000
+++ service/libertine_service/apt.py	2016-10-06 18:10:43 +0000
@@ -13,87 +13,43 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import apt
-import glob
-import os
 import re
 
-def _use_system_gpg():
-    """ Configures APT to use the system-wide GPG store.  Always."""
-    for key in 'Dir::Etc::Trusted', 'Dir::Etc::TrustedParts':
-        apt.apt_pkg.config.set(key, apt.apt_pkg.config.find_file(key))
-
-
-def _get_system_sources_list():
-    """Concatenates all the system apt/sources.list files into a single text buffer."""
-    sources_path = os.path.join('/etc', 'apt')
-    sources_list = glob.glob(os.path.join(sources_path, 'sources.list.d', '*.list'))
-    sources_list.append(os.path.join(sources_path, 'sources.list'))
-    sources = ''
-    for source in sources_list:
-        with open(source) as f:
-            sources += f.read()
-    return sources
-
-
-def _write_local_sources_list(rootdir):
-    """ Writes out a user-local sources.list file by copying all the systemwide ones. """
-    sources = _get_system_sources_list()
-    sources_path = os.path.join(rootdir, 'etc', 'apt')
-    os.makedirs(sources_path, exist_ok=True)
-    srcfile = os.path.join(sources_path, 'sources.list')
-    with open(srcfile, 'w') as f:
-        f.write(sources)
-    return srcfile
-
-
-def _get_local_apt_cache():
-    """ Gets a user-local APT cache. """
-    rootdir = os.path.join('/tmp', '.cache', 'libertine', 'apt')
-    srcfile = _write_local_sources_list(rootdir)
-
-    apt_cache = apt.Cache(rootdir=rootdir, memonly=True)
-    apt.apt_pkg.config.clear("APT::Update::Post-Invoke-Success")
-    apt_cache.update(sources_list=srcfile)
-    apt_cache.open()
-
-    return apt_cache
-
-
-def _get_apt_cache(config):
-    """ Factory function to get the APT cache obejct. """
-    _use_system_gpg()
-    if config.use_local_cache:
-        return _get_local_apt_cache()
-    else:
-        return apt.Cache()
+import gi
+gi.require_version('Libertine', '1')
+from gi.repository import Libertine
+
+from libertine import utils
+
 
 class AptCache(object):
     """docstring for AptCache"""
-    def __init__(self, config):
+    def __init__(self, container_id):
         super(AptCache, self).__init__()
-        self.config = config
+        self._cache = None
+        self._container = container_id
 
     def search(self, query):
+        self._load()
+        pkg_keys = [key for key in self._cache.keys() if re.match(query, key)]
         apps = []
-
-        cache = _get_apt_cache(self.config)
-        pkg_keys = [key for key in cache.keys() if re.match(query, key)]
         for key in pkg_keys:
             app = {}
-            app["name"] = cache[key].name
-            app["id"] = cache[key].name
-            app["package"] = cache[key].name
-            if len(cache[key].versions) > 0:
-                app["summary"] = cache[key].versions[0].summary
-                app["website"] = cache[key].versions[0].homepage
+            app["name"] = self._cache[key].name
+            app["id"] = self._cache[key].name
+            app["package"] = self._cache[key].name
+            if len(self._cache[key].versions) > 0:
+                app["summary"] = self._cache[key].versions[0].summary
+                app["website"] = self._cache[key].versions[0].homepage
             apps.append(app)
         return apps
 
     def app_info(self, app_id):
+        self._load()
+
         app_data = {}
-        cache = _get_apt_cache(self.config)
-        if app_id in cache:
-            app = cache[app_id]
+        if app_id in self._cache:
+            app = self._cache[app_id]
             app_data["name"] = app.name
             app_data["id"] = app.name
             app_data["package"] = app.name
@@ -104,3 +60,12 @@
             app_data["package"] = app.name
 
         return app_data
+
+    def _load(self):
+        if self._cache is None:
+            try:
+                utils.get_logger().debug("Trying aptcache for container %s" % self._container)
+                self._cache = apt.Cache(rootdir=Libertine.container_path(self._container))
+            except PermissionError:
+                utils.get_logger().debug("Trying system aptcache")
+                self._cache = apt.Cache()

=== modified file 'service/libertine_service/dbus.py'
--- service/libertine_service/dbus.py	2016-09-20 18:30:26 +0000
+++ service/libertine_service/dbus.py	2016-10-06 18:10:43 +0000
@@ -75,10 +75,10 @@
             raise
 
         try:
-            self.cache = appstream.AppStreamCache(config)
+            self.cache = appstream.AppStreamCache(CONTAINER_ID)
         except:
             log.warning("AppStream backend unavailable, falling back to Apt backend.")
-            self.cache = apt.AptCache(config)
+            self.cache = apt.AptCache(CONTAINER_ID)
 
         self.container = container.Container(log)
         self.progress = {}

=== modified file 'service/libertine_service/tasks.py'
--- service/libertine_service/tasks.py	2016-09-16 21:35:57 +0000
+++ service/libertine_service/tasks.py	2016-10-06 18:10:43 +0000
@@ -134,6 +134,10 @@
             self.log.debug("Creating container '%s' failed" % self.container)
             self.config.delete_container(self.container)
             return
+
+        container.install_package("appstream") # install appstream for searching packages
+        container.update_libertine_container() # update apt cache to generate appstream cache
+
         self.config.update_container_install_status(self.container, "ready")
 
     def _before(self):