Merge ~letterus/synapse-project:recoll_integration into synapse-project:master

[Johannes Brakensiek]

Add plugin that integrations searching via Recoll, originally written by Patrick Marchwiak.

This branch features the Recoll plugin originally written by Patrick Marchwiak at https://code.launchpad.net/~pmarchwiak/synapse-project/recoll-plugin/+merge/133784.

It was updated to the current API. In addition to the former behaviour a timeout for keystrokes was added. Thus a search query via Recoll only is performed if the user stopped typing.

This resolves the problem mentioned here: https://code.launchpad.net/~pmarchwiak/synapse-project/recoll-plugin/+merge/133784/comments/289583

Edit: In addition this plugin now only is going to start working after the third character entered. The assumption is you won't be able to enter any meaningful word using less than three chars.

[Johannes Brakensiek (letterus)]

Pinging this issue. Would be great if it could be merged.

Unmerged commits

df08c14... by Letterus <email address hidden> on 2022-12-31

More verbose code

4bfab77... by Letterus <email address hidden> on 2022-12-30

Disable debugging log

358f88f... by Letterus <email address hidden> on 2022-12-30

Add some async timer and timeouts to check if the user has stopped typing. Only start the Recoll search task then.

078ec2e... by Letterus <email address hidden> on 2022-12-30

First working version of Patrick Marchwiak's Recoll plugin reintegrated

diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 28a60b7..2827053 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -48,6 +48,7 @@ libplugins_la_VALASOURCES = \
 	pass-plugin.vala \
 	pastebin-plugin.vala \
 	pidgin-plugin.vala \
+	recoll-plugin.vala \
 	rhythmbox-plugin.vala \
 	selection-plugin.vala \
 	test-slow-plugin.vala \
diff --git a/src/plugins/recoll-plugin.vala b/src/plugins/recoll-plugin.vala
new file mode 100644
index 0000000..9bf64d7
--- /dev/null
+++ b/src/plugins/recoll-plugin.vala
@@ -0,0 +1,224 @@
+/*
+ *
+ * Authored by Patrick Marchwiak <[e-mail]>
+ * Updated by Johannes Brakensiek <[e-mail]>
+ *
+ */
+namespace Synapse
+{
+  
+  // Sends query to Recoll command line tool.
+  public class RecollPlugin : Object, Activatable, ItemProvider
+  {
+    // a mandatory property
+    public bool enabled { get; set; default = true; }
+
+    // properties for keeping track typing activity of the user
+    public signal void typing_finished ();
+    private bool user_typing = false;
+    private Timer timer;
+
+    const uint maxKeystrokeInterval = 300;
+
+    static construct
+    {
+      // register the plugin when the class is constructed
+      register_plugin ();
+    }
+
+    construct
+    {
+      timer = new Timer();
+    }
+  
+    // this method is called when a plugin is enabled
+    // use it to initialize your plugin
+    public void activate ()
+    {
+    }
+ 
+    // this method is called when a plugin is disabled
+    // use it to free the resources you're using
+    public void deactivate ()
+    {
+    }
+ 
+    // register your plugin in the UI
+    static void register_plugin ()
+    {
+      PluginRegistry.get_default ().register_plugin (
+        typeof (RecollPlugin),
+        _ ("Recoll"), // plugin title
+        _ ("Returns results of full text search against an existing Recoll index."), // description
+        "recoll", // icon name
+        register_plugin, // reference to this function
+        Environment.find_program_in_path ("recoll") != null, // true if user's system has all required components which the plugin needs
+        _ ("recoll is not installed") // error message
+      );
+    }
+ 
+    // an optional method to improve the speed of searches, 
+    // if you return false here, the search method won't be called
+    // for this query
+    public bool handles_query (Query query)
+    {
+      return (QueryFlags.FILES in query.query_type);
+    }
+
+    enum LineType {
+      FIELDS,
+      ABSTRACT_START,
+      ABSTRACT,
+      ABSTRACT_END;
+    }
+ 
+    private async void check_if_user_is_typing()
+    {
+      timer = new Timer();
+    
+      if(!user_typing) {
+        Timeout.add(50, () => {
+          ulong microseconds;          
+          user_typing = true;
+          timer.elapsed (out microseconds);
+          //print ("Milliseconds elapsed %lu\n", microseconds / 1000); 
+          if((microseconds / 1000) > maxKeystrokeInterval) {
+            user_typing = false;
+            check_if_user_is_typing.callback ();
+            return false;
+          }
+          return true;
+        });
+      }
+
+      yield;      
+
+      typing_finished ();
+    }
+   
+    private async void recollSearch (Query query, ResultSet results) throws SearchError
+    {
+      Pid pid;
+      int read_fd, write_fd;
+      string[] argv = {"recoll", 
+                       "-t",   // command line mode
+                       "-n",   // indices of results
+                       "0-20", // return first 20 results
+                       "-a",   // ALL TERMS mode
+                       "-A",   // output abstracts
+                       query.query_string};
+
+      try
+      {
+        Process.spawn_async_with_pipes (null, argv, null,
+                                        SpawnFlags.SEARCH_PATH,
+                                        null, out pid, out write_fd, out read_fd);
+        UnixInputStream read_stream = new UnixInputStream (read_fd, true);
+        DataInputStream recoll_output = new DataInputStream (read_stream);
+
+        // Sample output from `recoll -t` :
+        // ===============================
+        // Recoll query: ((kernel:(wqf=11) OR kernels OR kernelize OR kernelized))
+        // 4725 results (printing  1 max):
+        // text/plain	[file:///home/patrick/code/sample-results.txt]	[sample-results.txt]	8806	bytes	
+        // ABSTRACT
+        // some text summarizing the document usually has the keyword (kernel)
+        // /ABSTRACT
+      
+        string line = null;
+        var next_line_type = LineType.FIELDS;
+        
+        Utils.FileInfo result = null;
+        int line_idx = 0;
+        string description = null;
+        while ((line = yield recoll_output.read_line_async (Priority.DEFAULT)) != null) 
+        {
+          if (line_idx >= 2) // skip first two lines
+          {
+            if (next_line_type == LineType.FIELDS)
+            {
+              string[] fields = line.split("\t");
+              
+              //string mimetype = fields[0];
+              
+              string uri = fields[1];
+              uri = uri.substring(1, uri.length - 2);
+              
+              description = uri.split("://")[1];
+              
+              // FIXME: recoll already gives us the mimetype so FileInfo
+              // is doing extra work obtaining it from the file
+              result = new Utils.FileInfo (uri, typeof (UriMatch));
+              
+              yield result.initialize ();
+              
+              next_line_type = LineType.ABSTRACT_START;
+            }
+            else if (next_line_type == LineType.ABSTRACT_START)
+            {
+              // TODO check for the start of the abstract
+              next_line_type = LineType.ABSTRACT;
+            }
+            else if (next_line_type == LineType.ABSTRACT)
+            {
+              line = line.chug().chomp();
+              if (line != null && line != "")
+              { 
+                description = description + ": " + line;
+              }
+              next_line_type = LineType.ABSTRACT_END;
+            }
+            else if (next_line_type == LineType.ABSTRACT_END)
+            {
+              // TODO check for the end of the abstract
+              
+              // TODO use relevancy rating to set match score
+              
+              // score defaults to just under that of results from the locate plugin
+              result.match_obj.description = description;
+              results.add(result.match_obj, MatchScore.INCREMENT_MINOR * 2);
+              next_line_type = LineType.FIELDS;
+            }
+            else
+            {
+              // TODO handle unexpected output ?
+            }
+          }
+          line_idx++;
+        }
+      }
+      catch (Error err)
+      {
+        if (!query.is_cancelled ()) warning ("%s", err.message);
+      }
+    }
+
+    public async ResultSet? search (Query query) throws SearchError
+    {
+      check_if_user_is_typing.begin((obj,res) => {
+          check_if_user_is_typing.end(res);
+      });
+       
+      if (user_typing)
+      {
+        // wait
+        ulong signal_id = this.typing_finished.connect (() => {
+          search.callback ();
+        });
+        yield;
+        SignalHandler.disconnect (this, signal_id);
+      }
+
+      // make sure this method is called before returning any results
+      query.check_cancellable ();
+      if(query.query_string.char_count () < 3)
+          return null;
+
+      ResultSet results = new ResultSet ();
+      yield recollSearch(query, results);
+
+      query.check_cancellable ();
+      return results;
+    }
+  }
+}
diff --git a/src/ui/synapse-main.vala b/src/ui/synapse-main.vala
index 2d9c1d3..059e5c5 100644
--- a/src/ui/synapse-main.vala
+++ b/src/ui/synapse-main.vala
@@ -184,6 +184,7 @@ namespace Synapse
 #if HAVE_LIBREST
         typeof (ImgUrPlugin),
 #endif
+	typeof (RecollPlugin),
         // action-only plugins
         typeof (DevhelpPlugin),
         typeof (OpenSearchPlugin),