Merge lp:~max-rabkin/ibid/google-translate into lp:~ibid-core/ibid/old-trunk-1.6

[Max Rabkin]

[Max Rabkin (max-rabkin)]

Using a hard-coded language list means we can tell users which languages we support, and we can use a normal @match to extract the arguments. It also means we use Google's language codes, which are not the same as ISO 639-1: they use "iw" for Hebrew instead of "he" (but accept both), and other functions in the Language API use the ISO 639-2 code for Cherokee (there is no 639-1 code). Also, we can read the languages right there in the code, so no more surprises like Greek.

Of course, there are the usual disadvantages of hard-coded values: basically, when Google adds languages, we'll have to update the list.

[Stefano Rivera (stefanor)]

I see a conflict in the diff below.

[lp:~max-rabkin/ibid/google-translate]

827. By Max Rabkin on 2010-01-01

merge

[Max Rabkin (max-rabkin)]

On Thu, Dec 31, 2009 at 7:58 PM, Stefano Rivera <email address hidden> wrote:
> I see a conflict in the diff below.

Thanks, fixed.

[Stefano Rivera (stefanor)]

Your merge request also seems to modify ibid.ini

[lp:~max-rabkin/ibid/google-translate]

828. By Max Rabkin on 2010-01-02

undo changes to ibid.ini

[Max Rabkin (max-rabkin)]

> Your merge request also seems to modify ibid.ini

Gah, fixed.

[Stefano Rivera (stefanor)]

ibid/plugins/google.py:1: 'codecs' imported but unused
ibid/plugins/google.py:11: 'cacheable_download' imported but unused
ibid/plugins/google.py:13: 'get_html_parse_tree' imported but unused

Besides those I approve

review: Approve

[Michael Gorven (mgorven)]

Looks fine.
 review approve

review: Approve

[Jonathan Hitchcock (vhata)]

review: Approve

=== modified file 'ibid/plugins/google.py'
--- ibid/plugins/google.py	2009-12-30 21:21:20 +0000
+++ ibid/plugins/google.py	2010-01-02 11:17:13 +0000
@@ -9,6 +9,8 @@
 from ibid.plugins import Processor, match
 from ibid.config import Option, IntOption
 from ibid.utils import decode_htmlentities, json_webservice, cacheable_download
+from ibid.utils import human_join
+from ibid.utils.html import get_html_parse_tree
 
 help = {'google': u'Retrieves results from Google and Google Calculator.'}
 
@@ -135,24 +137,66 @@
 
     api_key = Option('api_key', 'Your Google API Key (optional)', None)
     referer = Option('referer', 'The referer string to use (API searches)', default_referer)
-    dest_lang = Option('dest_lang', 'Destination language when none is specified', 'en')
+    dest_lang = Option('dest_lang', 'Destination language when none is specified', 'english')
 
     chain_length = IntOption('chain_length', 'Maximum length of translation chains', 10)
 
-    @match(r'^translate\s+(.*)$')
-    def translate (self, event, data):
+    lang_names = {'afrikaans':'af', 'albanian':'sq', 'arabic':'ar',
+                  'belarusian':'be', 'bulgarian':'bg', 'catalan':'ca',
+                  'chinese':'zh', 'chinese simplified':'zh-cn',
+                  'chinese traditional':'zh-tw', 'croatian':'hr', 'czech':'cs',
+                  'danish':'da', 'dutch':'nl', 'english':'en', 'estonian':'et',
+                  'filipino':'tl', 'finnish':'fi', 'french':'fr',
+                  'galacian':'gl', 'german':'de', 'greek':'el', 'hebrew':'iw',
+                  'hindi':'hi', 'hungarian':'hu', 'icelandic':'is',
+                  'indonesian':'id', 'irish':'ga', 'italian':'it',
+                  'japanese':'ja', 'korean': 'ko', 'latvian':'lv',
+                  'lithuanian':'lt', 'macedonian':'mk', 'malay':'ms',
+                  'maltese':'mt', 'norwegian':'no', 'persian':'fa',
+                  'polish':'pl', 'portuguese':'pt', 'romanian':'ro',
+                  'russian': 'ru', 'serbian':'sr', 'slovak':'sk',
+                  'slovenian':'sl', 'spanish':'es', 'swahili':'sw',
+                  'swedish':'sv', 'thai':'th', 'turkish':'tr', 'ukrainian':'uk',
+                  'uzbek': 'uz', 'vietnamese':'vi', 'welsh':'cy',
+                  'yiddish':'yi'}
+
+    alt_lang_names = {'simplified':'zh-CN', 'simplified chinese':'zh-CN',
+                   'traditional':'zh-TW', 'traditional chinese':'zh-TW',
+                   'bokmal':'no', 'norwegian bokmal':'no',
+                   u'bokm\N{LATIN SMALL LETTER A WITH RING ABOVE}l':'no',
+                   u'norwegian bokm\N{LATIN SMALL LETTER A WITH RING ABOVE}l':
+                        'no',
+                   'farsi':'fa'}
+
+    LANG_REGEX = '|'.join(lang_names.keys() + lang_names.values() +
+                            alt_lang_names.keys())
+
+    @match(r'^(?:translation\s*)?languages$')
+    def languages (self, event):
+        event.addresponse(human_join(sorted(self.lang_names.keys())))
+
+    @match(r'^translate\s+(.*?)(?:\s+from\s+(' + LANG_REGEX + r'))?'
+            r'(?:\s+(?:in)?to\s+(' + LANG_REGEX + r'))?$')
+    def translate (self, event, text, src_lang, dest_lang):
+        dest_lang = self.language_code(dest_lang or self.dest_lang)
+        src_lang = self.language_code(src_lang or '')
+
         try:
-            translated = self._translate(event, *self._parse_request(data))[0]
+            translated = self._translate(event, text, src_lang, dest_lang)[0]
             event.addresponse(translated)
         except TranslationException, e:
             event.addresponse(u"I couldn't translate that: %s.", unicode(e))
 
-    @match(r'^translation[-\s]*(?:chain|party)\s+(.*)$')
-    def translation_chain (self, event, data):
+    @match(r'^translation[-\s]*(?:chain|party)\s+(.*?)'
+            r'(?:\s+from\s+(' + LANG_REGEX + r'))?'
+            r'(?:\s+(?:in)?to\s+(' + LANG_REGEX + r'))?$')
+    def translation_chain (self, event, phrase, src_lang, dest_lang):
         if self.chain_length < 1:
             event.addresponse(u"I'm not allowed to play translation games.")
         try:
-            phrase, src_lang, dest_lang = self._parse_request(data)
+            dest_lang = self.language_code(dest_lang or self.dest_lang)
+            src_lang = self.language_code(src_lang or '')
+
             chain = [phrase]
             for i in range(self.chain_length):
                 phrase, src_lang = self._translate(event, phrase,
@@ -167,38 +211,6 @@
         except TranslationException, e:
             event.addresponse(u"I couldn't translate that: %s.", unicode(e))
 
-    def _parse_request (self, data):
-        if not hasattr(self, 'lang_names'):
-            self._make_language_dict()
-
-        from_re = r'\s+from\s+(?P<from>[-()\s\w]+?)'
-        to_re = r'\s+(?:in)?to\s+(?P<to>[-()\s\w]+?)'
-
-        res = [(from_re, to_re), (to_re, from_re), (to_re,), (from_re,), ()]
-
-        # Try all possible specifications of source and target language until we
-        # find a valid one.
-        for pat in res:
-            pat = '(?P<text>.*)' + ''.join(pat) + '\s*$'
-            m = re.match(pat, data, re.IGNORECASE | re.UNICODE | re.DOTALL)
-            if m:
-                dest_lang = m.groupdict().get('to')
-                src_lang = m.groupdict().get('from')
-                try:
-                    if dest_lang:
-                        dest_lang = self.language_code(dest_lang)
-                    else:
-                        dest_lang = self.dest_lang
-
-                    if src_lang:
-                        src_lang = self.language_code(src_lang)
-                    else:
-                        src_lang = ''
-
-                    return (m.group('text'), src_lang, dest_lang)
-                except UnknownLanguageException:
-                    continue
-
     def _translate (self, event, phrase, src_lang, dest_lang):
         params = {
             'v': '1.0',
@@ -234,48 +246,22 @@
 
             raise TranslationException(msg)
 
-    def _make_language_dict (self):
-        self.lang_names = d = {}
-
-        filename = cacheable_download('http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt',
-                                        'google/ISO-639-2_utf-8.txt')
-        f = codecs.open(filename, 'rU', 'utf-8')
-        for line in f:
-            code2B, code2T, code1, englishNames, frenchNames = line.split('|')
-
-            # Identify languages by ISO 639-1 code if it exists; otherwise use
-            # ISO 639-2 (B). Google currently only translates languages with -1
-            # codes, but will may use -2 (B) codes in the future.
-            ident = code1 or code2B
-
-            d[code2B] = d[code2T] = d[code1] = ident
-            for name in englishNames.lower().split(';'):
-                d[name] = ident
-
-        del d['']
-
     def language_code (self, name):
-        """Convert a name to a language code.
-
-        Caller must call _make_language_dict first."""
+        """Convert a name to a language code."""
 
         name = name.lower()
 
-        m = re.match('^([a-z]{2})(?:-[a-z]{2})?$', name)
-        if m and m.group(1) in self.lang_names:
+        if name == '':
             return name
-        if 'simplified' in name:
-            return 'zh-CN'
-        if 'traditional' in name:
-            return 'zh-TW'
-        if re.search(u'bokm[a\N{LATIN SMALL LETTER A WITH RING ABOVE}]l', name):
-            # what Google calls Norwegian seems to be Bokmal
-            return 'no'
 
         try:
-            return self.lang_names[name]
+            return self.lang_names.get(name) or self.alt_lang_names[name]
         except KeyError:
-            raise UnknownLanguageException
+            m = re.match('^([a-z]{2,3})(?:-[a-z]{2})?$', name)
+            if m and m.group(1) in self.lang_names.values():
+                return name
+            else:
+                raise UnknownLanguageException
 
 # This Plugin uses code from youtube-dl
 # Copyright (c) 2006-2008 Ricardo Garcia Gonzalez