Merge lp:~jamesh/unity-lens-applications/index-local-scopes into lp:~unity-team/unity-lens-applications/libunity7-compatible

[James Henstridge]

Index local scopes and include them in the search results.

Index local scopes and include them in the search results.

This branch adds locally installed scopes to the search results from the applications scope.

If the software-center index includes scopes, it will also remove installed scopes from the "more suggestions" category of the results.

Most of the preview code is shared between local and remote scopes, with the enable/disable preview action updating the same dconf key (from chats with mhr3, this is the way we want to handle disabling of local scopes too).

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:345
http://jenkins.qa.ubuntu.com/job/unity-team-unity-lens-applications-libunity7-compatible-ci/6/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-team-unity-lens-applications-libunity7-compatible-raring-amd64-ci/6/console

Click here to trigger a rebuild:
http://s-jenkins:8080/job/unity-team-unity-lens-applications-libunity7-compatible-ci/6/rebuild

review: Needs Fixing (continuous-integration)

[lp:~jamesh/unity-lens-applications/index-local-scopes]

346. By James Henstridge on 2013-04-24

Merge from libunity7-compatible

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:346
http://jenkins.qa.ubuntu.com/job/unity-team-unity-lens-applications-libunity7-compatible-ci/7/
Executed test runs:
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-team-unity-lens-applications-libunity7-compatible-raring-amd64-ci/7

Click here to trigger a rebuild:
http://s-jenkins:8080/job/unity-team-unity-lens-applications-libunity7-compatible-ci/7/rebuild

review: Approve (continuous-integration)

[Michal Hruby (mhr3)]

There seem to be missing features here (local scopes are not displayed in "installed" category when doing empty search, sorting with the rest of the scopes, disabled local scopes in "more suggestions" category, etc), but what's in seems to work. +1

review: Approve

=== modified file 'configure.ac'
--- configure.ac	2013-03-13 10:13:14 +0000
+++ configure.ac	2013-04-24 12:06:28 +0000
@@ -77,6 +77,7 @@
                   zeitgeist-1.0 >= 0.3.8
                   libcolumbus0 >= 0.4.0
                   unity >= 6.7.0
+                  unity-protocol-private
                   libgnome-menu-3.0 >= 3.6.0)
 
 AC_SUBST(LENS_DAEMON_CFLAGS)
@@ -138,6 +139,12 @@
 AS_AC_EXPAND(DATADIR, $datarootdir)
 AC_SUBST(DATADIR)
 
+#####################################################
+# Look for protocol-private lib dir
+#####################################################
+PROTOCOLPRIVATELIBDIR="`$PKG_CONFIG --variable=libdir unity-protocol-private`/libunity"
+AC_SUBST(PROTOCOLPRIVATELIBDIR)
+
 #############################################
 # look for dbus service dir
 #############################################

=== modified file 'src/Makefile.am'
--- src/Makefile.am	2013-02-22 12:05:05 +0000
+++ src/Makefile.am	2013-04-24 12:06:28 +0000
@@ -35,6 +35,7 @@
   --pkg dee-1.0 \
   --pkg zeitgeist-1.0 \
   --pkg unity \
+  --pkg unity-protocol \
   --pkg gee-1.0 \
   --pkg libgnome-menu-3.0 \
   --pkg unity-package-search \
@@ -54,7 +55,7 @@
   unity-ratings-db.o \
   $(NULL)
 
-unity_applications_daemon_LDFLAGS = $(COVERAGE_LDFLAGS)
+unity_applications_daemon_LDFLAGS = -rpath $(PROTOCOLPRIVATELIBDIR) $(COVERAGE_LDFLAGS)
 
 unity_applications_daemon_VALASOURCES = \
   app-watcher.vala \

=== modified file 'src/config.vala.in'
--- src/config.vala.in	2010-06-22 07:36:19 +0000
+++ src/config.vala.in	2013-04-24 12:06:28 +0000
@@ -4,6 +4,8 @@
   
   const string DATADIR = "@DATADIR@";
   
+  const string PKGDATADIR = "@DATADIR@/unity";
+
   const string BINDIR = "@prefix@/bin";
   
   const string LOCALEDIR = "@DATADIR@/locale";
@@ -11,4 +13,4 @@
   const string PACKAGE = "@PACKAGE@";
   
   const string VERSION = "@VERSION@";
-}
\ No newline at end of file
+}

=== modified file 'src/daemon.vala'
--- src/daemon.vala	2013-04-23 15:01:12 +0000
+++ src/daemon.vala	2013-04-24 12:06:28 +0000
@@ -59,6 +59,7 @@
     /* The searcher for online material may be null if it fails to load
      * the Xapian index from the Software Center */
     private Unity.Package.Searcher? pkgsearcher;
+    private Unity.Package.Searcher? scopesearcher;
     public Unity.Package.Searcher appsearcher;
 
     /* Read the app ratings dumped by the Software Center */
@@ -167,6 +168,7 @@
       image_extensions.add ("jpg");
 
       build_app_menu_index ();
+      build_scope_index.begin ();
 
       file_icon_cache = new HashTable<string,Icon>(str_hash, str_equal);
       sc_mangler = new SoftwareCenterUtils.MangledDesktopFileLookup ();
@@ -434,6 +436,15 @@
       return false;
     }
 
+    private async void build_scope_index ()
+    {
+      var scope_registry = yield Unity.Protocol.ScopeRegistry.find_scopes (
+        Config.PKGDATADIR + "/scopes");
+
+      debug ("Indexing scopes");
+      scopesearcher = new Unity.Package.Searcher.for_scopes (scope_registry);
+    }
+
     private void populate_zg_templates ()
     {
       /* Create a template that activation of applications */
@@ -575,6 +586,13 @@
 
       transaction.commit ();
 
+      // add scopes (if filter is active)
+      if (!has_filter || type_filter.get_option ("scopes").active)
+      {
+        add_local_scopes_results (search_string, model, installed_uris);
+        add_remote_scopes_results (search_string, model);
+      }
+
       purchase_info = new PurchaseInfoHelper ();
 
       /* If we don't have a search we display 6 random apps */
@@ -639,12 +657,6 @@
         }
       }
 
-      // add scopes (if filter is active)
-      if (!has_filter || type_filter.get_option ("scopes").active)
-      {
-        add_remote_scopes_results (search_string, model);
-      }
-
       if (model.get_n_rows () == 0)
       {
         search.set_reply_hint ("no-results-hint",
@@ -1032,6 +1044,61 @@
       return null;
     }
 
+    private void add_local_scopes_results (string search_string,
+                                           Dee.Model model,
+                                           Set<string> installed_uris)
+    {
+      // If the local scopes Xapian index hasn't been built, return.
+      if (scopesearcher == null)
+        return;
+
+      bool has_search = !Utils.is_search_empty (search_string);
+
+      var results = scopesearcher.search (search_string, 0,
+                                          Unity.Package.SearchType.PREFIX,
+                                          has_search ?
+                                            Unity.Package.Sort.BY_RELEVANCY :
+                                            Unity.Package.Sort.BY_NAME);
+
+      foreach (unowned Unity.Package.PackageInfo info in results.results)
+      {
+        /* De-dupe by "application://foo.scope", since that is what is
+         * used for software-center results. */
+        string dedup_key = @"application://$(info.desktop_file)";
+        if (dedup_key in installed_uris)
+          continue;
+        installed_uris.add(dedup_key);
+
+        var uri = @"scope://$(info.desktop_file)";
+        var name = info.application_name;
+        var icon_hint = info.icon;
+        var comment = info.description;
+
+        try
+        {
+          Icon base_icon = icon_hint == null || icon_hint == "" ?
+            get_default_scope_icon () : Icon.new_for_string (icon_hint);
+          var anno_icon = new AnnotatedIcon (base_icon);
+          anno_icon.size_hint = IconSizeHint.SMALL;
+          icon_hint = anno_icon.to_string ();
+        }
+        catch (Error err)
+        {
+          icon_hint = "";
+        }
+
+        model.append (uri,
+                      icon_hint,
+                      Category.INSTALLED,
+                      ResultType.DEFAULT,
+                      "application/x-unity-scope",
+                      name != "" ? name : uri,
+                      comment,
+                      "", // dnd_uri ?!
+                      empty_asv);
+      }
+    }
+
     private void add_remote_scopes_results (string search_string,
                                             Dee.Model model)
     {
@@ -1040,7 +1107,7 @@
       {
         unowned string scope_id =
           remote_scopes_model.get_string (iter, RemoteScopesColumn.SCOPE_ID);
-        var uri = "scope://%s".printf (scope_id);
+        var uri = @"scope://$(scope_id)";
         var name =
           remote_scopes_model.get_string (iter, RemoteScopesColumn.NAME);
         var icon_hint = 
@@ -1413,18 +1480,23 @@
 
         // figure out if the scope is remote
         bool is_remote_scope = false;
+        var info = scopesearcher.get_by_desktop_file (scope_id);
+
         Dee.ModelIter found_iter = null;
-        var iter = remote_scopes_model.get_first_iter ();
-        var end_iter = remote_scopes_model.get_last_iter ();
-        while (iter != end_iter)
+        if (info == null)
         {
-          if (remote_scopes_model.get_string (iter, 0) == scope_id)
+          var iter = remote_scopes_model.get_first_iter ();
+          var end_iter = remote_scopes_model.get_last_iter ();
+          while (iter != end_iter)
           {
-            is_remote_scope = true;
-            found_iter = iter;
-            break;
+            if (remote_scopes_model.get_string (iter, 0) == scope_id)
+            {
+              is_remote_scope = true;
+              found_iter = iter;
+              break;
+            }
+            iter = remote_scopes_model.next (iter);
           }
-          iter = remote_scopes_model.next (iter);
         }
 
         if (is_remote_scope)
@@ -1454,9 +1526,36 @@
                                                   description,
                                                   icon,
                                                   screenshot);
+        }
+        else if (info != null)
+        {
+          var name = info.application_name;
+          if (name == null || name == "")
+            name = info.desktop_file;
+          // XXX: add this to package info.
+          var description = info.description;
+          Icon? icon = null;
+          try
+          {
+            if (info.icon != null && info.icon != "")
+              icon = Icon.new_for_string (info.icon);
+          }
+          catch (Error err)
+          {
+            warning ("%s", err.message);
+          }
+          if (icon == null)
+            icon = get_default_scope_icon ();
+
+          preview = new Unity.ApplicationPreview (name,
+                                                  "",
+                                                  description,
+                                                  icon,
+                                                  null);
+        }
+        if (preview != null) {
+          PreviewAction action;
           preview.set_rating(-1.0f, 0);
-
-          PreviewAction action;
           if (scope_disabled)
           {
             action = new Unity.PreviewAction ("enable-scope", _("Enable"), null);

=== modified file 'src/unity-package-search.cc'
--- src/unity-package-search.cc	2013-04-16 12:13:32 +0000
+++ src/unity-package-search.cc	2013-04-24 12:06:28 +0000
@@ -239,6 +239,9 @@
       if (g_app_info_get_display_name (app_info))
         doc.add_value (XAPIAN_VALUE_APPNAME, g_app_info_get_display_name (app_info));
 
+      if (g_app_info_get_description (app_info))
+        doc.add_value (XAPIAN_VALUE_DESCRIPTION, g_app_info_get_description (app_info));
+
       if (g_app_info_get_icon (app_info))
       {
         gchar *icon_str = g_icon_to_string (g_app_info_get_icon (app_info));
@@ -383,6 +386,95 @@
   return searcher;
 }
 
+/* Recursively traverse the set of scopes */
+static void
+index_scope (Xapian::WritableDatabase                *db,
+             Xapian::TermGenerator                   *indexer,
+             UnityProtocolScopeRegistryScopeMetadata *info)
+{
+  Xapian::Document doc;
+  char *dum1;
+
+  if (info->name != NULL)
+    doc.add_value (XAPIAN_VALUE_APPNAME, info->name);
+  if (info->description != NULL)
+    doc.add_value (XAPIAN_VALUE_DESCRIPTION, info->description);
+  if (info->icon != NULL)
+    doc.add_value (XAPIAN_VALUE_ICON, info->icon);
+  if (info->id != NULL)
+    doc.add_value (XAPIAN_VALUE_DESKTOP_FILE, info->id);
+
+  indexer->set_document (doc);
+  if (info->name != NULL)
+  {
+    dum1 = unity_applications_lens_utils_preprocess_string (info->name);
+    indexer->index_text (dum1, 5);
+    g_free (dum1);
+  }
+  if (info->description != NULL)
+  {
+    dum1 = unity_applications_lens_utils_preprocess_string (info->description);
+    indexer->index_text (dum1, 5);
+    g_free (dum1);
+  }
+  for (GSList *l = info->keywords; l != NULL; l = l->next)
+  {
+    dum1 = unity_applications_lens_utils_preprocess_string ((char *)l->data);
+    indexer->index_text (dum1, 0);
+    indexer->index_text (dum1, 0, "KW");
+    g_free (dum1);
+  }
+
+  /* Always assume Type=Scope for scopes... */
+  doc.add_term ("ATscope");
+
+  /* Index application name */
+  dum1 = g_strconcat ("AA", info->name, NULL);
+  doc.add_term (dum1);
+  g_free (dum1);
+
+  db->add_document (doc);
+}
+
+UnityPackageSearcher*
+unity_package_searcher_new_for_scopes (UnityProtocolScopeRegistry *scope_registry)
+{
+  UnityPackageSearcher *searcher;
+  Xapian::WritableDatabase *db;
+
+  searcher = new UnityPackageSearcher;
+  db = new Xapian::WritableDatabase ();
+  searcher->db = db;
+  searcher->db->add_database (Xapian::InMemory::open ());
+
+  init_searcher (searcher);
+
+  /* Index the menu recursively */
+  searcher->db_merged = false;
+  Xapian::TermGenerator *indexer = new Xapian::TermGenerator ();
+
+  GSList *scopes = unity_protocol_scope_registry_get_scopes (scope_registry);
+  for (GSList *l = scopes; l != NULL; l = l->next)
+  {
+    UnityProtocolScopeRegistryScopeRegistryNode *node = (UnityProtocolScopeRegistryScopeRegistryNode *)l->data;
+    index_scope (db, indexer, node->scope_info);
+
+    // Also index any sub-scopes
+    for (GSList *sl = node->sub_scopes; sl != NULL; sl = sl->next)
+    {
+      UnityProtocolScopeRegistryScopeMetadata *info = (UnityProtocolScopeRegistryScopeMetadata *)sl->data;
+      index_scope (db, indexer, info);
+    }
+  }
+  delete indexer;
+  db->flush ();
+
+  searcher->matcher = new Columbus::Matcher();
+  buildMatcher(searcher);
+
+  return searcher;
+}
+
 /* Create a new searcher that searches into the Xapian index
  * provided by the Software Center */
 UnityPackageSearcher*
@@ -452,6 +544,9 @@
   string appname = doc.get_value (XAPIAN_VALUE_APPNAME);
   pkginfo->application_name = g_strdup (appname.c_str ());
 
+  string description = doc.get_value (XAPIAN_VALUE_DESCRIPTION);
+  pkginfo->description = g_strdup (description.c_str ());
+
   string desktop_file = doc.get_value (XAPIAN_VALUE_DESKTOP_FILE);
   pkginfo->desktop_file = g_strdup (desktop_file.c_str ());
 
@@ -892,14 +987,20 @@
     Xapian::Document doc = searcher->db->get_document (*it);
     string value = doc.get_value (XAPIAN_VALUE_DESKTOP_FILE);
 
+    bool matches = false;
     size_t sep = value.find (':');
     if (sep != string::npos)
     {
-      if (value.compare (sep+1, value.size() - sep, query) == 0)
-      {
-        pkginfo = _pkginfo_from_document (doc);
-        return pkginfo;
-      }
+      matches = value.compare (sep+1, value.size() - sep, query) == 0;
+    }
+    else
+    {
+      matches = value == query;
+    }
+    if (matches)
+    {
+      pkginfo = _pkginfo_from_document (doc);
+      return pkginfo;
     }
     ++it;
   }

=== modified file 'src/unity-package-search.h'
--- src/unity-package-search.h	2012-10-09 15:26:49 +0000
+++ src/unity-package-search.h	2013-04-24 12:06:28 +0000
@@ -19,6 +19,7 @@
 
 #include <glib.h>
 #include <gmenu-tree.h>
+#include <unity-protocol.h>
 
 typedef struct _UnityPackageSearcher UnityPackageSearcher;
 
@@ -44,6 +45,7 @@
 {
         gchar    *package_name;
         gchar    *application_name;
+        gchar    *description;
         gchar    *desktop_file;
         gchar    *icon;
         gchar    *price;
@@ -61,6 +63,7 @@
 UnityPackageSearcher*     unity_package_searcher_new    ();
 
 UnityPackageSearcher*     unity_package_searcher_new_for_menu    (GMenuTree *menu);
+UnityPackageSearcher*     unity_package_searcher_new_for_scopes  (UnityProtocolScopeRegistry *scope_registry);
 
 void                      unity_package_searcher_free   (UnityPackageSearcher *searcher);
 

=== modified file 'src/xapian-utils.vala'
--- src/xapian-utils.vala	2013-03-22 10:10:13 +0000
+++ src/xapian-utils.vala	2013-04-24 12:06:28 +0000
@@ -109,9 +109,9 @@
     if (Utils.is_search_empty (search_string))
     {
       if (options == null || !options.filtering)
-        return "type:Application";
+        return "(type:Application OR type:Scope)";
       else
-        return "type:Application AND " + extract_type_query (options);
+        return "(type:Application OR type:Scope) AND " + extract_type_query (options);
     }
     else
     {
@@ -127,9 +127,9 @@
       s = s.delimit ("-", ' ');
       
       if (options == null || !options.filtering)
-        return "type:Application AND " + s;
+        return "(type:Application OR type:Scope) AND " + s;
       else
-        return "type:Application AND %s AND %s".printf (extract_type_query (options), s);
+        return "(type:Application OR type:Scope) AND %s AND %s".printf (extract_type_query (options), s);
     }
   }
 }

=== modified file 'tests/unit/Makefile.am'
--- tests/unit/Makefile.am	2013-03-01 15:41:08 +0000
+++ tests/unit/Makefile.am	2013-04-24 12:06:28 +0000
@@ -30,6 +30,8 @@
 	$(test_libs) \
 	$(UNITY_PACKAGE_SEARCH_LIBS)
 
+LDFLAGS += -rpath $(PROTOCOLPRIVATELIBDIR)
+
 AM_CPPFLAGS = \
   $(LENS_DAEMON_CFLAGS) \
   -I$(srcdir) \

=== modified file 'tests/unit/test-xapian-utils.vala'
--- tests/unit/test-xapian-utils.vala	2012-10-08 08:41:45 +0000
+++ tests/unit/test-xapian-utils.vala	2013-04-24 12:06:28 +0000
@@ -43,7 +43,7 @@
   internal static void test_empty_search_query ()
   {
     var query = XapianUtils.prepare_pkg_search_string ("", null);
-    assert (query == "type:Application");
+    assert (query == "(type:Application OR type:Scope)");
   }
 
   internal static void test_empty_search_with_single_cat_filter ()
@@ -54,7 +54,7 @@
     filter.add_option ("internet", "Internet", null);
 
     var query = XapianUtils.prepare_pkg_search_string ("", filter);
-    assert (query == "type:Application AND (category:AudioVideo)");
+    assert (query == "(type:Application OR type:Scope) AND (category:AudioVideo)");
   }
 
   internal static void test_empty_search_with_multi_cat_filter ()
@@ -65,16 +65,16 @@
     filter.add_option ("internet", "Internet", null);
     filter.add_option ("game", "Game", null).active = true;
     var query = XapianUtils.prepare_pkg_search_string ("", filter);
-    assert (query == "type:Application AND (category:Game OR category:AudioVideo)");
+    assert (query == "(type:Application OR type:Scope) AND (category:Game OR category:AudioVideo)");
   }
 
   internal static void test_search_query_no_filters ()
   {
     var query = XapianUtils.prepare_pkg_search_string (" foo ", null);
-    assert (query == "type:Application AND foo");
+    assert (query == "(type:Application OR type:Scope) AND foo");
 
     query = XapianUtils.prepare_pkg_search_string ("foo", null);
-    assert (query == "type:Application AND foo");
+    assert (query == "(type:Application OR type:Scope) AND foo");
   }
 
   internal static void test_search_query_with_single_cat_filter ()
@@ -84,7 +84,7 @@
     filter.add_option ("media", "Media", null);
     filter.add_option ("internet", "Internet", null).active = true;
     var query = XapianUtils.prepare_pkg_search_string ("foo", filter);
-    assert (query == "type:Application AND (category:Network) AND foo");
+    assert (query == "(type:Application OR type:Scope) AND (category:Network) AND foo");
   }
 
   internal static void test_search_query_with_multi_cat_filter ()
@@ -95,7 +95,7 @@
     filter.add_option ("internet", "Internet", null).active = true;
     filter.add_option ("game", "Game", null).active = true;
     var query = XapianUtils.prepare_pkg_search_string ("foo", filter);
-    assert (query == "type:Application AND (category:Game OR category:Network) AND foo");
+    assert (query == "(type:Application OR type:Scope) AND (category:Game OR category:Network) AND foo");
   }
 
   internal static void test_zg_empty_search_query ()

=== modified file 'vapi/unity-package-search.deps'
--- vapi/unity-package-search.deps	2012-10-18 12:36:37 +0000
+++ vapi/unity-package-search.deps	2013-04-24 12:06:28 +0000
@@ -1,2 +1,3 @@
 glib-2.0
 libgnome-menu-3.0
+unity-protocol

=== modified file 'vapi/unity-package-search.vapi'
--- vapi/unity-package-search.vapi	2012-10-08 15:11:07 +0000
+++ vapi/unity-package-search.vapi	2013-04-24 12:06:28 +0000
@@ -27,6 +27,8 @@
       public Searcher ();
       [CCode (cname = "unity_package_searcher_new_for_menu")]
       public Searcher.for_menu(GMenu.Tree menu);
+      [CCode (cname = "unity_package_searcher_new_for_scopes")]
+      public Searcher.for_scopes(Unity.Protocol.ScopeRegistry scope_registry);
       public SearchResult search (string search_string, uint max_hits, Unity.Package.SearchType search_type, Unity.Package.Sort sort);
       public SearchResult get_random_apps (string? filter_query, uint n_apps);
       public SearchResult get_apps (string? filter_query, uint n_apps, AppFilterCallback cb);
@@ -49,6 +51,7 @@
       public PackageInfo ();
       public string package_name;
       public string application_name;
+      public string description;
       public string desktop_file;
       public string icon;
       public string price;