Merge lp:~lifeless/launchpad/malone into lp:launchpad

[Robert Collins]

Avoid perfoming hundreds of thousands of comparisons as a prelude to actual searching - the fti index is the right place to perform relevance assessment.

Do less work searching, so searching is faster. Faster searching -> less load -> less timeouts. We all win. Winning is good. Goodness is good.

[Paul Hummer (rockstar)]

8 - extra_constraints_tables=None):
9 + extra_constraints_tables=None,
10 + fast_enabled=True):
11 + """Return the tsearch2 query that should be use to do a phrase search.
12 +
13 + The precise heuristics applied by this function will vary as we tune
14 + the system.
15 +

*used

review: Approve (code)

=== modified file 'lib/canonical/database/nl_search.py'
--- lib/canonical/database/nl_search.py	2009-06-25 05:30:52 +0000
+++ lib/canonical/database/nl_search.py	2010-07-22 05:49:45 +0000
@@ -36,7 +36,58 @@
 
 
 def nl_phrase_search(phrase, table, constraints='',
-                     extra_constraints_tables=None):
+                     extra_constraints_tables=None,
+                     fast_enabled=True):
+    """Return the tsearch2 query that should be used to do a phrase search.
+
+    The precise heuristics applied by this function will vary as we tune
+    the system.
+
+    It is the interface by which a user query should be turned into a backend
+    search language query.
+
+    Caveats: The model class must define a 'fti' column which is then used used
+    for full text searching.
+
+    :param phrase: A search phrase.
+    :param table: This should be the SQLBase class representing the base type.
+    :param constraints: Additional SQL clause that limits the rows to a subset
+        of the table.
+    :param extra_constraints_tables: A list of additional table names that are
+        needed by the constraints clause.
+    :param fast_enabled: If true use a fast, but less precise, code path. When
+        feature flags are available this will be converted to a feature flag.
+    :return: A tsearch2 query string.
+    """
+    terms = nl_term_candidates(phrase)
+    if len(terms) == 0:
+        return ''
+    if fast_enabled:
+        return _nl_phrase_search(terms, table, constraints,
+            extra_constraints_tables)
+    else:
+        return _slow_nl_phrase_search(terms, table, constraints,
+            extra_constraints_tables)
+
+
+def _nl_phrase_search(terms, table, constraints, extra_constraints_tables):
+    """Perform a very simple pruning of the phrase, letting fti do ranking.
+
+    See nl_phrase_search for the contract of this function.
+    """
+    bad_words = set(['firefox', 'ubuntu', 'launchpad'])
+    terms = set(terms)
+    filtered_terms = []
+    for term in terms:
+        if term.lower() in bad_words:
+            continue
+        filtered_terms.append(term)
+    # sorted for testing convenience.
+    return '|'.join(sorted(filtered_terms))
+
+
+def _slow_nl_phrase_search(terms, table, constraints,
+    extra_constraints_tables):
     """Return the tsearch2 query that should be use to do a phrase search.
 
     This function implement an algorithm similar to the one used by MySQL
@@ -56,7 +107,7 @@
     closer in the text at the top of the list, while still returning rows that
     use only some of the terms.
 
-    :phrase: A search phrase.
+    :terms: Some candidate search terms.
 
     :table: This should be the SQLBase class representing the base type.
 
@@ -66,14 +117,12 @@
     :extra_constraints_tables: A list of additional table names that are
     needed by the constraints clause.
 
-    Caveat: The SQLBase class must define a 'fti' column .
-    This is the column that is used for full text searching.
+    Caveat: The model class must define a 'fti' column which is then used
+    for full text searching.
     """
     total = table.select(
         constraints, clauseTables=extra_constraints_tables).count()
-    term_candidates = nl_term_candidates(phrase)
-    if len(term_candidates) == 0:
-        return ''
+    term_candidates = terms
     if total < 5:
         return '|'.join(term_candidates)
 

=== modified file 'lib/canonical/launchpad/doc/textsearching.txt'
--- lib/canonical/launchpad/doc/textsearching.txt	2009-07-23 17:49:31 +0000
+++ lib/canonical/launchpad/doc/textsearching.txt	2010-07-22 05:49:45 +0000
@@ -588,7 +588,12 @@
 
 To get the actual tsearch2 query that should be run, you will use the
 nl_phrase_search() function. This one takes two mandatory parameters and
-two optional ones. You pass in the search phrase and a SQLObject class.
+two optional ones. You pass in the search phrase and a database model class.
+
+The original nl_phrase_search has proved slow, so there are now two
+implementations in the core.
+
+First we describe the slow implementation.
 
 The select method of that class will be use to count the number of rows
 that is matched by each term. Term matching 50% or more of the total
@@ -608,7 +613,8 @@
 
 So firefox will be removed from the final query:
 
-    >>> nl_phrase_search('system is slow when running firefox', Question)
+    >>> nl_phrase_search('system is slow when running firefox', Question,
+    ...     fast_enabled=False)
     u'system|slow|run'
 
     >>> nl_term_candidates('how do I do this?')
@@ -616,6 +622,12 @@
     >>> nl_phrase_search('how do I do this?', Question)
     ''
 
+The fast code path only removes firefox, ubuntu and launchpad today:
+
+    >>> nl_phrase_search('system is slow when running firefox on ubuntu',
+    ...     Question)
+    u'run|slow|system'
+
 
 ==== Using other constraints ====
 
@@ -640,13 +652,13 @@
     >>> nl_phrase_search(
     ...     'firefox gets very slow on flickr', Question,
     ...     "Question.product = %s AND Product.active = 't'" % firefox_product.id,
-    ...     ['Product'])
+    ...     ['Product'], fast_enabled=False)
     u'slow|flickr'
 
 When the query only has stop words or common words in it, the returned
 query will be the empty string:
 
-    >>> nl_phrase_search('firefox will not do it', Question)
+    >>> nl_phrase_search('ubuntu will not do it', Question)
     ''
 
 When there are no candidate rows, only stemming and stop words removal
@@ -656,7 +668,7 @@
     0
     >>> nl_phrase_search('firefox is very slow on flickr', Question,
     ...                  'product = -1')
-    u'firefox|slow|flickr'
+    u'flickr|slow'
 
 
 ==== No keywords filtering with few rows ====
@@ -693,4 +705,4 @@
     ...     'firefox is slow', Question,
     ...     'distribution = %s AND sourcepackagename = %s' % sqlvalues(
     ...     ubuntu, firefox_package.sourcepackagename))
-    u'firefox|slow'
+    u'slow'

=== modified file 'lib/lp/answers/doc/faqtarget.txt'
--- lib/lp/answers/doc/faqtarget.txt	2010-03-18 18:55:02 +0000
+++ lib/lp/answers/doc/faqtarget.txt	2010-07-22 05:49:45 +0000
@@ -159,6 +159,7 @@
     >>> for faq in target.findSimilarFAQs('How do I use the Answer Tracker'):
     ...     print faq.title
     How to answer a question
+    How to use bug mail
     How to become a Launchpad king
 
 The results are ordered by relevancy. The first document is considered

=== modified file 'lib/lp/answers/doc/questiontarget.txt'
--- lib/lp/answers/doc/questiontarget.txt	2010-04-08 20:24:52 +0000
+++ lib/lp/answers/doc/questiontarget.txt	2010-07-22 05:49:45 +0000
@@ -359,6 +359,11 @@
     ...     print t.title
     New question
     Another question
+    Question title3
+    Question title2
+    Question title1
+    Question title0
+
 
 
 Answer contacts

=== modified file 'lib/lp/answers/stories/question-add-in-other-languages.txt'
--- lib/lp/answers/stories/question-add-in-other-languages.txt	2009-11-11 22:17:17 +0000
+++ lib/lp/answers/stories/question-add-in-other-languages.txt	2010-07-22 05:49:45 +0000
@@ -45,8 +45,6 @@
     ...     row.first('a').renderContents()
     'Installation of Java Runtime Environment for Mozilla'
     'Problema al recompilar kernel con soporte smp (doble-n\xc3\xbacleo)'
-    'mailto: problem in webpage'
-    '\xd8\xb9\xd9\x83\xd8\xb3 ...'
 
     >>> for tag in find_tags_by_class(browser.contents, 'warning message'):
     ...     print tag.renderContents()

=== modified file 'lib/lp/answers/stories/question-add.txt'
--- lib/lp/answers/stories/question-add.txt	2009-11-17 02:33:27 +0000
+++ lib/lp/answers/stories/question-add.txt	2010-07-22 05:49:45 +0000
@@ -83,9 +83,6 @@
     Installation of Java Runtime Environment for Mozilla  (Answered)
         posted on 2006-07-20 by  Sample Person
         in ...mozilla-firefox... package in Ubuntu
-    mailto: problem in webpage (Solved)
-        posted on 2006-07-20 by Sample Person, answered by Foo Bar
-        in ...mozilla-firefox... package in Ubuntu
     >>> similar_questions.a['href']
     u'http://answers.../ubuntu/+source/mozilla-firefox/+question/...'
 

=== modified file 'lib/lp/bugs/doc/bugtask.txt'
--- lib/lp/bugs/doc/bugtask.txt	2010-06-23 13:31:35 +0000
+++ lib/lp/bugs/doc/bugtask.txt	2010-07-22 05:49:45 +0000
@@ -1375,6 +1375,13 @@
     >>> for similar_bug in sorted(similar_bugs, key=attrgetter('id')):
     ...     print "%s: %s" % (similar_bug.id, similar_bug.title)
     1: Firefox does not support SVG
+    2: Blackhole Trash folder
+    9: Thunderbird crashes
+    10: another test bug
+    16: a test bug
+    17: a test bug
+    18: New Bug
+
 
 ... and for SourcePackages.
 
@@ -1387,6 +1394,7 @@
     >>> for similar_bug in sorted(similar_bugs, key=attrgetter('id')):
     ...     print "%s: %s" % (similar_bug.id, similar_bug.title)
     1: Firefox does not support SVG
+    18: New Bug
 
 Private bugs won't show up in the list of similar bugs unless the user
 is a direct subscriber. We'll demonstrate this by creating a new bug

=== modified file 'lib/lp/bugs/model/bugtask.py'
--- lib/lp/bugs/model/bugtask.py	2010-07-08 13:10:41 +0000
+++ lib/lp/bugs/model/bugtask.py	2010-07-22 05:49:45 +0000
@@ -29,6 +29,7 @@
 
 from storm.expr import And, Alias, AutoTables, In, Join, LeftJoin, Or, SQL
 from storm.sqlobject import SQLObjectResultSet
+from storm.store import EmptyResultSet
 from storm.zope.interfaces import IResultSet, ISQLObjectResultSet
 
 import pytz
@@ -1462,6 +1463,8 @@
     def findSimilar(self, user, summary, product=None, distribution=None,
                     sourcepackagename=None):
         """See `IBugTaskSet`."""
+        if not summary:
+            return EmptyResultSet()
         # Avoid circular imports.
         from lp.bugs.model.bug import Bug
         search_params = BugTaskSearchParams(user)
@@ -1482,9 +1485,6 @@
         else:
             raise AssertionError('Need either a product or distribution.')
 
-        if not summary:
-            return BugTask.select('1 = 2')
-
         search_params.fast_searchtext = nl_phrase_search(
             summary, Bug, ' AND '.join(constraint_clauses), ['BugTask'])
         return self.search(search_params)
@@ -2069,7 +2069,6 @@
             ', '.join(tables), ' AND '.join(clauses))
         return clause
 
-
     def search(self, params, *args):
         """See `IBugTaskSet`."""
         store_selector = getUtility(IStoreSelector)

=== modified file 'lib/lp/bugs/stories/guided-filebug/xx-sorting-by-relevance.txt'
--- lib/lp/bugs/stories/guided-filebug/xx-sorting-by-relevance.txt	2009-07-15 13:22:29 +0000
+++ lib/lp/bugs/stories/guided-filebug/xx-sorting-by-relevance.txt	2010-07-22 05:49:45 +0000
@@ -20,9 +20,6 @@
     #4 Reflow problems with complex page layouts
     New (0 comments) last updated 2006-07-14...
     Malone pages that use more complex layouts...
-    #5 Firefox install instructions should be complete
-    New (0 comments) last updated 2006-07-14...
-    All ways of downloading firefox should provide complete...
 
 If we instead enter a summary that matches bug #4 better, the result will
 be reversed.