Merge lp:~mhr3/zeitgeist-extensions/scheme-detection into lp:zeitgeist-extensions

[Michal Hruby]

[lp:~mhr3/zeitgeist-extensions/scheme-detection]

76. By Michal Hruby on 2011-09-30

Get rid of unnecessary branches in the check_index_and_start_worker method

77. By Michal Hruby on 2011-09-30

Try to prevent some races

[Mikkel Kamstrup Erlandsen (kamstrup)]

Ok. The code looks good. I tested the hell out of this with log in/out cycles. Deleting random parts of the Xapian index, and calling ForceReindex() over DBus while concurrently doing searches. Everything was rock solid.

review: Approve

=== modified file 'fts/fts.py'
--- fts/fts.py	2011-09-07 08:42:40 +0000
+++ fts/fts.py	2011-09-30 09:44:25 +0000
@@ -62,6 +62,7 @@
 log = logging.getLogger("zeitgeist.fts")
 
 INDEX_FILE = os.path.join(constants.DATA_PATH, "fts.index")
+INDEX_VERSION = "1"
 INDEX_LOCK = threading.Lock()
 FTS_DBUS_OBJECT_PATH = "/org/gnome/zeitgeist/index/activity"
 FTS_DBUS_INTERFACE = "org.gnome.zeitgeist.Index"
@@ -314,28 +315,24 @@
 		
 		This method should be called from the main thread and only once.
 		It starts the worker thread as a side effect.
+
+		We are clearing the queue, because there may be a race when an
+		event insertion / deletion is already queued and our index
+		is corrupted. Creating a new queue instance should be safe,
+		because we're running in main thread as are the index_event
+		and delete_event methods, and the worker thread wasn't yet
+		started.
 		"""
-		if "XAPIAN_CJK_NGRAM" in os.environ and self._index.get_metadata("cjk_ngram") != "1":
-			# If the database was built prior to CJK support
-			# force of a reindex
-			log.info("Index built without CJK support. Upgrading index")
+		if self._index.get_metadata("fts_index_version") != INDEX_VERSION:
+			log.info("Index must be upgraded. Doing full rebuild")
+			self._queue = Queue(0)
 			self._queue.put(Reindex(self._engine))
 		elif self._index.get_doccount() == 0:
 			# If the index is empty we trigger a rebuild
 			# We must delay reindexing until after the engine is done setting up
 			log.info("Empty index detected. Doing full rebuild")
+			self._queue = Queue(0)
 			self._queue.put(Reindex(self._engine))
-		else:
-			# If the index doesn't use the zgsu prefix, it must be old-style,
-			# and we must rebuild it
-			query = self._query_parser.parse_query ("zgsu:file*",
-		                                            self.QUERY_PARSER_FLAGS)
-			self._enquire.set_query (query)
-			hits = self._enquire.get_mset (0, 1)
-			hit_count = hits.get_matches_estimated()
-			if hit_count == 0:
-				log.info ("Old index format detected. Rebuilding index.")
-				self._queue.put(Reindex(self._engine))
 		
 		# Now that we've checked the index from the main thread we can start the worker
 		self._worker.start()
@@ -472,10 +469,8 @@
 		self._index = xapian.WritableDatabase(INDEX_FILE, xapian.DB_CREATE_OR_OVERWRITE)
 		self._query_parser.set_database (self._index)
 		self._enquire = xapian.Enquire(self._index)
-		
-		# Register that this index was built with CJK enabled
-		if "XAPIAN_CJK_NGRAM" in os.environ :
-			self._index.set_metadata("cjk_ngram", "1")
+	
+		self._index.set_metadata("fts_index_version", INDEX_VERSION)
 		
 		log.info("Preparing to rebuild index with %s events" % len(event_list))
 		for e in event_list : self._queue.put(e)
@@ -607,7 +602,7 @@
 		# usually web URIs, are indexed in another way because there may
 		# be domain name etc. in there we want to rank differently
 		scheme, host, path = self._split_uri (url_unescape (uri))
-		if scheme == "file://" or not scheme:
+		if scheme == "file" or not scheme:
 			path, name = os.path.split(path)
 			self._tokenizer.index_text(name, 5)
 			self._tokenizer.index_text(name, 5, "N")
@@ -617,9 +612,9 @@
 			while path and name:
 				weight = weight / 1.5
 				path, name = os.path.split(path)
-				self._tokenizer.index_text(name, weight)
+				self._tokenizer.index_text(name, int(weight))
 			
-		elif scheme == "mailto:":
+		elif scheme == "mailto":
 			tokens = host.split("@")
 			name = tokens[0]
 			self._tokenizer.index_text(name, 6)