Merge lp:~mvo/software-center/de-duplication-multiple-pkgnames-lp1043159 into lp:software-center

[Michael Vogt]

This branch ensures that duplicatied packages are eliminated from the
get_query_for_pkgnames() if there are multiple documents in the database.
This can happen when a pacakge is available via the apt-xapian-index and
also via the software-center-agent (e.g. packages from extras.ubuntu.com)

[Gary Lasker (gary-lasker)]

Hi Michael, thanks for this branch. Please see my comment in MP https://code.launchpad.net/~mvo/software-center/exact-match-duplication-lp891613/+merge/126902 that describes a strange side-effect that I noticed. It occurs when this branch is merged also. Thanks!

[Gary Lasker (gary-lasker)]

Hi Michael, when running this branch I'm hitting two exceptions, one at startup as shown:

Traceback (most recent call last):
  File "/home/tremolux/Projects/quantal/software-center_de-duplication_multiple_pkgnames_lp1043159_tweak/software-center/softwarecenter/db/categories.py", line 188, in _recommend_me_result
    self.query = self.db.get_query_for_pkgnames(pkgs)
AttributeError: 'RecommendedForYouCategory' object has no attribute 'db'

And the other when viewing the app details view:

Traceback (most recent call last):
  File "/home/tremolux/Projects/quantal/software-center_de-duplication_multiple_pkgnames_lp1043159_tweak/software-center/softwarecenter/db/categories.py", line 230, in _recommend_app_result
    self.query = self.db.get_query_for_pkgnames(pkgs)
AttributeError: 'AppRecommendationsCategory' object has no attribute 'db'

It seems we are just missing passing the db into a couple of classes. I made a small branch that should take care of these issues, and also updates the unit tests per the changes. Please check it over and feel free to use it if it looks right to you:

  lp:~gary-lasker/software-center/de-duplication-multiple-pkgnames-lp1043159-tweak

I will approve the branch in any case, please feel free to merge with the needed fix (or I can do it tomorrow).

Many thanks!

review: Approve

=== modified file 'softwarecenter/db/categories.py'
--- softwarecenter/db/categories.py	2012-09-27 13:02:26 +0000
+++ softwarecenter/db/categories.py	2012-09-28 11:48:24 +0000
@@ -42,7 +42,6 @@
 from softwarecenter.backend.recagent import RecommenderAgent
 from softwarecenter.db.appfilter import AppFilter
 from softwarecenter.db.enquire import AppEnquire
-from softwarecenter.db.utils import get_query_for_pkgnames
 from softwarecenter.region import get_region_cached
 from softwarecenter.utils import utf8
 
@@ -183,10 +182,10 @@
             pkgs.append(item['package_name'])
         if self.subcategory:
             self.query = xapian.Query(xapian.Query.OP_AND,
-                                  get_query_for_pkgnames(pkgs),
+                                  self.db.get_query_for_pkgnames(pkgs),
                                   self.subcategory.query)
         else:
-            self.query = get_query_for_pkgnames(pkgs)
+            self.query = self.db.get_query_for_pkgnames(pkgs)
         self.emit("needs-refresh")
 
     def _recommender_agent_error(self, recommender_agent, msg):
@@ -227,7 +226,7 @@
         pkgs = []
         for item in result_list['data']:
             pkgs.append(item['package_name'])
-        self.query = get_query_for_pkgnames(pkgs)
+        self.query = self.db.get_query_for_pkgnames(pkgs)
         self.emit("needs-refresh")
 
     def _recommender_agent_error(self, recommender_agent, msg):

=== modified file 'softwarecenter/db/database.py'
--- softwarecenter/db/database.py	2012-09-14 13:40:18 +0000
+++ softwarecenter/db/database.py	2012-09-28 11:48:24 +0000
@@ -24,13 +24,12 @@
 import threading
 import xapian
 from softwarecenter.db.application import Application
-from softwarecenter.db.utils import get_query_for_pkgnames
 from softwarecenter.db.pkginfo import get_pkg_info
 import softwarecenter.paths
 
 from gi.repository import GObject, Gio
 
-#from softwarecenter.utils import *
+from softwarecenter.utils import ExecutionTime
 from softwarecenter.enums import (
     AVAILABLE_FOR_PURCHASE_MAGIC_CHANNEL_NAME,
     PkgStates,
@@ -302,6 +301,28 @@
         assert popcon_max > 0
         return popcon_max
 
+    def get_query_for_pkgnames(self, pkgnames):
+        """ return a xapian query that matches exactly the list of pkgnames """
+        enquire = xapian.Enquire(self.xapiandb)
+        query = xapian.Query()
+        for pkgname in pkgnames:
+            # even on the raspbery pi this query super quick (~0.003s)
+            with ExecutionTime("de-dup query_for_pkgnames"):
+                tmp_query = xapian.Query("AP" + pkgname)
+                enquire.set_query(tmp_query)
+                result = enquire.get_mset(0, 1)
+            # see bug #1043159, we need to ensure that we de-duplicate
+            # when there is a pkg and a app (e.g. from the s-c-agent) in the db
+            if len(result) == 1:
+                query = xapian.Query(xapian.Query.OP_OR,
+                                     query,
+                                     xapian.Query("AP" + pkgname))
+            else:
+                query = xapian.Query(xapian.Query.OP_OR,
+                                     query,
+                                     xapian.Query("XP" + pkgname))
+        return query
+
     def get_query_list_from_search_entry(self, search_term,
         category_query=None):
         """ get xapian.Query from a search term string and a limit the
@@ -343,7 +364,7 @@
         search_term = search_term.strip()
         # get a pkg query
         if "," in search_term:
-            pkg_query = get_query_for_pkgnames(search_term.split(","))
+            pkg_query = self.get_query_for_pkgnames(search_term.split(","))
         else:
             pkg_query = xapian.Query()
             for term in search_term.split():

=== modified file 'softwarecenter/db/enquire.py'
--- softwarecenter/db/enquire.py	2012-09-28 11:48:24 +0000
+++ softwarecenter/db/enquire.py	2012-09-28 11:48:24 +0000
@@ -160,7 +160,7 @@
             if exact_pkgname_query:
                 with ExecutionTime("de-duplication"):
                     q_app = xapian.Query(terms[0].replace("XP", "AP"))
-                    nr_apps, nr_pkgs =  self._get_estimate_nr_apps_and_nr_pkgs(
+                    nr_apps, nr_pkgs = self._get_estimate_nr_apps_and_nr_pkgs(
                         enquire, q_app, xfilter)
                     if nr_apps == 1:
                         q = q_app

=== modified file 'softwarecenter/db/utils.py'
--- softwarecenter/db/utils.py	2012-09-14 07:12:33 +0000
+++ softwarecenter/db/utils.py	2012-09-28 11:48:24 +0000
@@ -18,7 +18,6 @@
 
 import logging
 import os
-import xapian
 
 from gi.repository import GObject
 
@@ -45,19 +44,6 @@
     GObject.child_watch_add(pid, _on_update_software_center_agent_finished)
 
 
-def get_query_for_pkgnames(pkgnames):
-    """ return a xapian query that matches exactly the list of pkgnames """
-    query = xapian.Query()
-    for pkgname in pkgnames:
-        query = xapian.Query(xapian.Query.OP_OR,
-                             query,
-                             xapian.Query("XP" + pkgname))
-        query = xapian.Query(xapian.Query.OP_OR,
-                             query,
-                             xapian.Query("AP" + pkgname))
-    return query
-
-
 def get_installed_apps_list(db):
     """ return a list of installed applications """
     apps = set()

=== modified file 'softwarecenter/ui/gtk3/views/lobbyview.py'
--- softwarecenter/ui/gtk3/views/lobbyview.py	2012-09-24 09:49:23 +0000
+++ softwarecenter/ui/gtk3/views/lobbyview.py	2012-09-28 11:48:24 +0000
@@ -46,7 +46,6 @@
                                           CategoriesParser,
                                           get_category_by_name,
                                           categories_sorted_by_name)
-from softwarecenter.db.utils import get_query_for_pkgnames
 from softwarecenter.distro import get_distro
 from softwarecenter.backend.scagent import SoftwareCenterAgent
 from softwarecenter.backend.reviews import get_review_loader
@@ -131,7 +130,7 @@
             app = Application("", pkgs[0])
             self.emit("application-activated", app)
         else:
-            query = get_query_for_pkgnames(pkgs)
+            query = self.db.get_query_for_pkgnames(pkgs)
             title = exhibit.title_translated
             untranslated_name = exhibit.package_names
             # create a temp query

=== modified file 'tests/test_database.py'
--- tests/test_database.py	2012-09-28 11:48:24 +0000
+++ tests/test_database.py	2012-09-28 11:48:24 +0000
@@ -45,8 +45,9 @@
     get_installed_apps_list,
 )
 from softwarecenter.enums import (
+    NonAppVisibility,
+    PkgStates,
     XapianValues,
-    PkgStates,
     )
 from softwarecenter.region import (
     REGION_BLACKLIST_TAG,
@@ -866,32 +867,45 @@
 ]
 """
 
-    def test_search_app_pkgname_duplication_lp891613(self):
-        # create a fake database to simualte a run of software-center-agent
+    @classmethod
+    def setUpClass(cls):
         cache = get_pkg_info()
         cache.open()
         db = xapian.WritableDatabase(TEST_DB,
                                      xapian.DB_CREATE_OR_OVERWRITE)
-        res = update_from_json_string(
-            db, cache, self.APP_INFO_JSON, origin="local")
+        update_from_json_string(db, cache, cls.APP_INFO_JSON, origin="local")
         db.close()
-        self.assertTrue(res)
+
+    def setUp(self):
+        # create a fake database to simualte a run of software-center-agent
         # create a StoreDatabase and add our other db
-        db = get_test_db()
-        db.add_database(xapian.Database(TEST_DB))
-        db.open(use_axi=True)
-        enquire = AppEnquire(db._aptcache, db)
+        self.db = get_test_db()
+        self.db.add_database(xapian.Database(TEST_DB))
+        self.db.open(use_axi=True)
+        self.enquire = AppEnquire(self.db._aptcache, self.db)
+        
+    def test_search_app_pkgname_duplication_lp891613(self):
         # simulate a pkg "apt" that is both in the agent and in the x-a-i db
         search_term = "apt"
-        search_query = db.get_query_list_from_search_entry(search_term)
-        enquire.set_query(search_query, nonblocking_load=False)
-        self.assertTrue(len(enquire._matches) > 2)
-        for m in enquire._matches:
+        search_query = self.db.get_query_list_from_search_entry(search_term)
+        self.enquire.set_query(search_query, nonblocking_load=False)
+        self.assertTrue(len(self.enquire._matches) > 2)
+        for m in self.enquire._matches:
             doc = m.document
             # ensure that all hits are "apps" and do not come from a-x-i
             self.assertNotEqual(
                 doc.get_value(XapianValues.PKGNAME), "")
 
+    def test_search_custom_pkgs_list_lp1043159(self):
+        # simulate a pkg "apt" that is both in the agent and in the x-a-i db
+        pkgs = ["apt","gedit"]
+        search_query = self.db.get_query_for_pkgnames(pkgs)
+        self.enquire.set_query(search_query, 
+                               # custom package lists are always in this mode
+                               nonapps_visible=NonAppVisibility.ALWAYS_VISIBLE,
+                               nonblocking_load=False)
+        self.assertEqual(len(self.enquire._matches), 2)
+
 
 if __name__ == "__main__":
     #import logging