Merge lp:~max-rabkin/ibid/wordlist into lp:ibid

[Max Rabkin]

[Stefano Rivera (stefanor)]

I'm a little worried that "is ... in ..." is too generic, but we can see how it works.

WARNING:plugins.unicode:Found a non-unicode string: "There's no such word"

It would be nice if it worked out the box. You suggested /usr/share/dict/words on IRC...

review: Approve

[lp:~max-rabkin/ibid/wordlist]

1033. By Max Rabkin on 2011-07-26

Unicode string literals

1034. By Max Rabkin on 2011-07-26

Use /usr/share/dict/words as a wordlist

[Max Rabkin (max-rabkin)]

This needs some usability work: better syntax, default wordlist, and a way to list wordlists.

review: Needs Fixing

[Jonathan Hitchcock (vhata)]

Too broad a match. We already have horribly conflicting plugins that match when they shouldn't.

review: Needs Fixing

Unmerged revisions

1034. By Max Rabkin on 2011-07-26

Use /usr/share/dict/words as a wordlist

1033. By Max Rabkin on 2011-07-26

Unicode string literals

1032. By Max Rabkin on 2011-07-26

Wordlist processor with globs and anagrams

=== modified file 'ibid/plugins/languages.py'
--- ibid/plugins/languages.py	2010-05-15 14:30:56 +0000
+++ ibid/plugins/languages.py	2011-07-26 20:51:28 +0000
@@ -1,6 +1,9 @@
 # Copyright (c) 2008-2010, Michael Gorven, Stefano Rivera, Max Rabkin
 # Released under terms of the MIT/X/Expat Licence. See COPYING for details.
 
+import codecs
+from collections import defaultdict
+import fnmatch
 from random import choice
 import re
 from urllib import urlencode
@@ -9,9 +12,9 @@
 from dictclient import Connection
 
 from ibid.plugins import Processor, match
-from ibid.config import Option, IntOption
+from ibid.config import Option, IntOption, DictOption
 from ibid.utils import decode_htmlentities, json_webservice, human_join, \
-                        is_url, iri_to_uri
+                        is_url, iri_to_uri, plural
 
 features = {}
 
@@ -284,4 +287,90 @@
             else:
                 raise UnknownLanguageException
 
+features['wordlist'] = {
+    'description': u'Check whether words exist in a wordlist, '
+                    'and search for anagrams or using wildcards.',
+    'categories': ('lookup',),
+}
+
+class Wordlist(Processor):
+    wordlists = DictOption('wordlists',
+                           u"A mapping from wordlist names to files",
+                           {'words': '/usr/share/dict/words'})
+    usage = u"""is <word> in <wordlist>?
+                words matching <glob> in <wordlist>
+                [sub]anagrams of <glob> in <wordlist>"""
+    features = ('wordlist',)
+
+    def get_wordlist(self, event, wordlist):
+        try:
+            for name, filename in self.wordlists.iteritems():
+                if wordlist.lower() == name.lower():
+                    return codecs.open(filename, 'rU', 'utf-8')
+            else:
+                event.addresponse(u"I don't have a wordlist called %s", wordlist)
+                return None
+        except IOError:
+            event.addresponse(u"I can't open that wordlist")
+            return None
+
+    def answer(self, event, words, wordlist):
+        if not words:
+            event.addresponse(u"There's no such word")
+        else:
+            # Sort by decreasing word length, but don't use reverse=True because
+            # we want to retain alphabetical order by stability
+            words.sort(key=lambda w: -len(w))
+            event.addresponse(u"%(words)s %(are)s in %(wordlist)s",
+                                {'words': human_join(words),
+                                 'are': plural(len(words), "is", "are"),
+                                 'wordlist': wordlist})
+
+    @match('(?:is |words matching )?{glob:chunk} in {wordlist:any}')
+    def search(self, event, glob, wordlist):
+        f = self.get_wordlist(event, wordlist)
+        if f is None:
+            return
+
+        words = fnmatch.filter((w.strip().upper() for w in f), glob.upper())
+        self.answer(event, words, wordlist)
+
+    @match('(?P<subanagrams>sub)?(?:an|anagrams(?: of)?) {glob:chunk} in {wordlist:any}')
+    def anagram(self, event, subanagrams, glob, wordlist):
+        for forbidden in '[]*':
+            if forbidden in glob:
+                event.addresponse(u'Only letters and ? are allowed in anagram '
+                                  'searches')
+                return
+
+        f = self.get_wordlist(event, wordlist)
+        if f is None:
+            return
+
+        glob = glob.upper()
+        globcount = defaultdict(int)
+        for char in glob:
+            globcount[char] += 1
+
+        n = len(glob)
+        results = []
+        for word in f:
+            word = word.strip().upper()
+            l = len(word)
+            if l > n or (l < n and not subanagrams):
+                continue
+
+            count = defaultdict(int)
+            blanks = 0
+            for char in word:
+                count[char] += 1
+                if count[char] > globcount[char]:
+                    blanks += 1
+                    if blanks > globcount['?']:
+                        break
+            else:
+                results.append(word)
+
+        self.answer(event, results, wordlist)
+
 # vi: set et sta sw=4 ts=4: