Merge lp:~renamer-developers/renamer/improve-tv-filename-parser into lp:renamer

[Jonathan Jacobs]

Replace PyParsing with PyMeta and implement metadata overrides for the "tvrage" command.

I'm wondering if I should put the grammar in a separate file and commit the result of the "generate_parser" script to the repo, the advantage would be improved import times.

[Tristan Seligmann (mithrandi)]

review: Approve

[lp:~renamer-developers/renamer/improve-tv-filename-parser]

90. By Jonathan Jacobs on 2010-10-26

Fix misnamed test method.

91. By Jonathan Jacobs on 2010-10-27

Use precompiled filename grammar when available.

92. By Jonathan Jacobs on 2010-10-27

Script to generate precompiled grammar.

93. By Jonathan Jacobs on 2010-10-27

Damnit, pyflakes!

=== modified file 'DEPS'
--- DEPS	2010-10-24 11:06:45 +0000
+++ DEPS	2010-10-27 00:51:02 +0000
@@ -1,6 +1,6 @@
 Axiom 0.6.0
 Epsilon 0.5.0
 Mutagen 1.15
-PyParsing 1.5.1
+PyMeta 0.4.0
 Python 2.5
 Twisted 2.5.0

=== modified file 'renamer/plugins/tv.py'
--- renamer/plugins/tv.py	2010-10-15 08:51:27 +0000
+++ renamer/plugins/tv.py	2010-10-27 00:51:02 +0000
@@ -1,20 +1,77 @@
 import string
 import urllib
 
+try:
+    import pymeta
+    from pymeta.grammar import OMeta
+    from pymeta.runtime import ParseError
+    pymeta # Ssssh, Pyflakes.
+except ImportError:
+    pymeta = None
+
 from twisted.web.client import getPage
 
-try:
-    import pyparsing
-    from pyparsing import (
-        alphanums, nums, Word, Literal, ParseException, SkipTo, FollowedBy,
-        ZeroOrMore, Combine, NotAny, Optional, StringEnd)
-    pyparsing # Ssssh, Pyflakes.
-except ImportError:
-    pyparsing = None
-
 from renamer import logging
 from renamer.plugin import RenamingCommand
 from renamer.errors import PluginError
+try:
+    from renamer._compiled_grammar.tv import Parser as FilenameGrammar
+    FilenameGrammar # Ssssh, Pyflakes.
+except ImportError:
+    FilenameGrammar = None
+
+
+
+filenameGrammar = """
+complete_strict    ::= <series_strict>:series <separator> <episode_strict>:episode
+                    => series, episode
+complete_lenient   ::= <series_lenient>:series <separator> <episode_lenient>:episode
+                    => series, episode
+partial_silly      ::= <series_silly>:series <separator> <episode_silly>:episode
+                    => series, episode
+only_episode_silly ::= <episode_silly>:episode
+                    => None, episode
+only_episode       ::= <episode_strict>:episode
+                    => None, episode
+only_series        ::= (<series_word>:word <separator> => word)+:words
+                    => ' '.join(words), [None, None]
+
+separator          ::= <hard_separator> | <soft_separator>
+soft_separator     ::= '.' | ' ' | '-' | '_'
+hard_separator     ::= ('_' '-' '_'
+                       |' ' '-' ' '
+                       |'.' '-' '.')
+
+series_strict      ::= (<series_word>:word <separator> ~(<episode_strict> <separator>) => word)*:words <series_word>:word
+                      => ' '.join(words + [word])
+series_lenient     ::= (<series_word>:word <separator> ~(<episode_lenient> <separator>) => word)*:words <series_word>:word
+                      => ' '.join(words + [word])
+series_silly       ::= (<series_word>:word <soft_separator> ~(<episode_silly> <separator>) => word)*:words <separator>
+                      => ' '.join(words)
+series_word        ::= (<letter> | <digit>)+:name => ''.join(name)
+
+episode_strict     ::= (<episode_x> | <episode_x2> | <episode_lettered>):ep
+                      => map(''.join, ep)
+episode_lenient    ::= (<episode_strict> | <episode_numbers>):ep
+                      => map(''.join, ep)
+episode_silly      ::= <digit>+:ep
+                      => map(''.join, [ep, ep])
+
+episode_lettered   ::= ('S' | 's') <digit>+:season ('E' | 'e') <digit>+:episode
+                      => season, episode
+episode_numbers    ::= <digit>:a <digit>:b <digit>:c <digit>?:d
+                      => ([a, b], [c, d]) if d else ([a], [b, c])
+episode_x          ::= <digit>+:season 'x' <digit>+:episode
+                      => season, episode
+episode_x2         ::= '[' <digit>+:season 'x' <digit>+:episode ']'
+                    => season, episode
+"""
+
+
+
+if FilenameGrammar is None:
+    class FilenameGrammar(OMeta.makeGrammar(filenameGrammar, globals())):
+        pass
 
 
 
@@ -39,56 +96,16 @@
         '$series [${season}x${padded_episode}] - $title')
 
 
+    optParameters = [
+        ('series',  None, None, 'Override series name.'),
+        ('season',  None, None, 'Override season number.', int),
+        ('episode', None, None, 'Override episode number.', int)]
+
+
     def postOptions(self):
-        self.filenameParser = self._createParser()
-
-
-    def _createParser(self):
-        """
-        Create the filename parser.
-        """
-        if pyparsing is None:
+        if pymeta is None:
             raise PluginError(
-                'The "pyparsing" package is required for this command')
-
-        def L(value):
-            return Literal(value).suppress()
-
-        number = Word(nums)
-        digit = Word(nums, exact=1)
-
-        separator = ( Literal('_-_')
-                    | Literal(' - ')
-                    | Literal('.-.')
-                    | Literal('-')
-                    | Literal('.')
-                    | Literal('_')
-                    | Literal(' '))
-        separator = separator.suppress().leaveWhitespace()
-
-        season = number.setResultsName('season')
-        exact_season = Word(nums, exact=2).setResultsName('season')
-        short_season = digit.setResultsName('season')
-        epnum = number.setResultsName('ep')
-        exact_epnum = Word(nums, exact=2).setResultsName('ep')
-        episode = ( season + L('x') + epnum
-                  | L('[') + season + L('x') + epnum + L(']')
-                  | L('S') + season + L('E') + epnum
-                  | L('s') + season + L('e') + epnum
-                  | exact_season + exact_epnum
-                  | short_season + exact_epnum)
-
-        series_word = Word(alphanums)
-        series = ZeroOrMore(
-            series_word + separator + NotAny(episode + separator)) + series_word
-        series = Combine(series, joinString=' ').setResultsName('series_name')
-
-        extension = '.' + Word(alphanums).setResultsName('ext') + StringEnd()
-
-        title = SkipTo(FollowedBy(extension))
-
-        return (series + separator + episode + Optional(separator + title) +
-                extension)
+                'The "pymeta" package is required for this command')
 
 
     def buildMapping(self, (seriesName, season, episode, episodeName)):
@@ -101,20 +118,49 @@
             title=episodeName)
 
 
-    def extractParts(self, filename):
+    def extractParts(self, filename, overrides=None):
         """
         Get TV episode information from a filename.
         """
-        try:
-            parse = self.filenameParser.parseString(filename)
-        except ParseException, e:
-            raise PluginError(
-                'No patterns could be found in "%s" (%r)' % (filename, e))
-        else:
-            parts = parse.series_name, parse.season, parse.ep, parse.ext
-            logging.msg('Found parts in "%s": %r' % (filename, parts),
-                        verbosity=4)
-            return parts
+        if overrides is None:
+            overrides = {}
+
+        rules = ['complete_strict', 'complete_lenient']
+        # We can only try the partial rules if there are some overrides.
+        if filter(None, overrides.values()):
+            rules.extend([
+                'only_episode',
+                'partial_silly',
+                'only_series',
+                'only_episode_silly'])
+
+        for rule in rules:
+            g = FilenameGrammar(filename)
+            logging.msg('Trying grammar rule "%s"' % (rule,),
+                        verbosity=5)
+            try:
+                res, err = g.apply(rule)
+            except ParseError, e:
+                try:
+                    logging.msg('Parsing error:', verbosity=5)
+                    for line in (e.formatError(filename).strip()).splitlines():
+                        logging.msg(line, verbosity=5)
+                except:
+                    pass
+                continue
+            else:
+                series, (season, episode) = res
+                parts = (
+                    overrides.get('series') or series,
+                    overrides.get('season') or season,
+                    overrides.get('episode') or episode)
+                if None not in parts:
+                    logging.msg('Found parts in "%s": %r' % (filename, parts),
+                                verbosity=4)
+                    return parts
+
+        raise PluginError(
+            'No patterns could be found in "%s"' % (filename))
 
 
     def extractMetadata(self, pageData):
@@ -147,8 +193,8 @@
     # IRenamerCommand
 
     def processArgument(self, arg):
-        # XXX: why does our pattern care about the extension?
-        seriesName, season, episode, ext = self.extractParts(arg.basename())
+        seriesName, season, episode = self.extractParts(
+            arg.basename(), overrides=self)
         d = self.lookupMetadata(seriesName, season, episode)
         d.addCallback(self.buildMapping)
         return d

=== modified file 'renamer/test/test_tvrage.py'
--- renamer/test/test_tvrage.py	2010-10-01 15:45:57 +0000
+++ renamer/test/test_tvrage.py	2010-10-27 00:51:02 +0000
@@ -22,27 +22,25 @@
     Tests for L{renamer.plugins.tv.TVRage}.
     """
     cases = [
-        ('Profiler - S01E01 - Insight.avi', 'Profiler', '01', '01', 'avi'),
-        ('Heroes [1x01] - Genesis.avi', 'Heroes', '1', '01', 'avi'),
-        ('Heroes S01E10 HDTV XviD.avi', 'Heroes', '01', '10', 'avi'),
-        ('heroes.108.hdtv-lol.avi', 'heroes', '1', '08', 'avi'),
-        ('arrested.development.302.avi', 'arrested development', '3', '02', 'avi'),
-        ('Heroes.S01E11.HDTV.XviD-K4RM4.avi', 'Heroes', '01', '11', 'avi'),
-        ('How I Met Your Mother - 101 - Pilot.avi', 'How I Met Your Mother', '1', '01', 'avi'),
-        ('24.s6e4.dvdrip.xvid-aerial.avi', '24', '6', '4', 'avi'),
-        ('harsh.realm.-.1x01.-.pilot.avi', 'harsh realm', '1', '01', 'avi'),
-        ('DayBreak_S01E09.avi', 'DayBreak', '01', '09', 'avi'),
-        ('Xena - 2x05 - Return of Callisto.avi', 'Xena', '2', '05', 'avi'),
-        ('Sliders_-_4x22_Revelations_(divx).avi', 'Sliders', '4', '22', 'avi'),
-        ('Xena_4x02_Adventures In The Sin Trade - Part 2.avi', 'Xena', '4', '02', 'avi'),
-        ('Sliders 501 - The Unstuck Man.avi', 'Sliders', '5', '01', 'avi'),
-        ('buffy.2x03.dvdrip.xvid-tns.avi', 'buffy', '2', '03', 'avi'),
-        # XXX: This is broken and probably has been for a long time, it would
-        # be nice if it worked again.
-        #('the.4400.1x05.avi', 'the 4400', '1', '05', 'avi'),
-        # This should work, but doesn't.
-        #('flash.gordon.2007.s01e02.dvdrip.xvid-reward.avi', 'flash gordon 2007', '01', '02', 'avi'),
-        ('ReGenesis - 1x13.avi', 'ReGenesis', '1', '13', 'avi')]
+        ('Profiler - S01E01 - Insight.avi', 'Profiler', '01', '01'),
+        ('Heroes [1x01] - Genesis.avi', 'Heroes', '1', '01'),
+        ('Heroes S01E10 HDTV XviD.avi', 'Heroes', '01', '10'),
+        ('heroes.108.hdtv-lol.avi', 'heroes', '1', '08'),
+        ('arrested.development.302.avi', 'arrested development', '3', '02'),
+        ('Heroes.S01E11.HDTV.XviD-K4RM4.avi', 'Heroes', '01', '11'),
+        ('How I Met Your Mother - 101 - Pilot.avi', 'How I Met Your Mother', '1', '01'),
+        ('24.s6e4.dvdrip.xvid-aerial.avi', '24', '6', '4'),
+        ('harsh.realm.-.1x01.-.pilot.avi', 'harsh realm', '1', '01'),
+        ('DayBreak_S01E09.avi', 'DayBreak', '01', '09'),
+        ('Xena - 2x05 - Return of Callisto.avi', 'Xena', '2', '05'),
+        ('Sliders_-_4x22_Revelations_(divx).avi', 'Sliders', '4', '22'),
+        ('Xena_4x02_Adventures In The Sin Trade - Part 2.avi', 'Xena', '4', '02'),
+        ('Sliders 501 - The Unstuck Man.avi', 'Sliders', '5', '01'),
+        ('buffy.2x03.dvdrip.xvid-tns.avi', 'buffy', '2', '03'),
+        ('the.4400.1x05.avi', 'the 4400', '1', '05'),
+        ('flash.gordon.2007.s01e02.dvdrip.xvid-reward.avi', 'flash gordon 2007', '01', '02'),
+        ('Foo - 508 - The cat has 9 lives.avi', 'Foo', '5', '08'),
+        ('ReGenesis - 1x13.avi', 'ReGenesis', '1', '13')]
 
 
     def setUp(self):
@@ -71,17 +69,66 @@
             self.plugin.extractParts, 'thiswillnotwork')
 
 
-    def test_missingPyParsing(self):
-        """
-        Attempting to use the TV Rage plugin without PyParsing installed raises
-        a L{renamer.errors.PluginError}.
-        """
-        self.patch(tv, 'pyparsing', None)
+    def test_extractPartsWithOverrides(self):
+        """
+        Override parts take preference when extracting TV show information from
+        filenames.
+        """
+        overrideCases = [
+            dict(series='House'),
+            dict(season=51),
+            dict(episode=99),
+            dict(series='House', season=51),
+            dict(series='House', episode=99),
+            dict(series='House', season=51, episode=99)]
+
+        for overrides in overrideCases:
+            for case in self.cases:
+                expected = (
+                    overrides.get('series', case[1]),
+                    overrides.get('season', case[2]),
+                    overrides.get('episode', case[3]))
+                self.assertEquals(
+                    self.plugin.extractParts(case[0], overrides=overrides),
+                    expected)
+
+
+    def test_extractPartsPartial(self):
+        """
+        When providing overrides partial information can be extracted from
+        filenames and combined with the override values.
+        """
+        cases = [
+            ('s01e02.avi', 'House', '01', '02', dict(series='House')),
+            ('House - 1.avi', 'House', 3, '1', dict(season=3)),
+            ('Arrested Development - 1.avi', 'Arrested Development', '1', 2, dict(episode=2)),
+            ('Chuck.avi', 'Chuck', 1, 2, dict(season=1, episode=2)),
+            ('Stargate SG1.avi', 'Stargate SG1', 1, 2, dict(season=1, episode=2)),
+            ('How I Met Your Mother.avi', 'How I Met Your Mother', 1, 2, dict(season=1, episode=2)),
+            ('1.avi', 'House', 1, '1', dict(series='House', season=1)),
+            ('1.avi', 'House', '1', 2, dict(series='House', episode=2))]
+
+        self.assertRaises(errors.PluginError,
+            self.plugin.extractParts, 's01e01.avi')
+
+        for filename, series, season, episode, overrides in cases:
+            self.plugin.extractParts(filename, overrides=overrides),
+            self.assertEquals(
+                self.plugin.extractParts(filename, overrides=overrides),
+                (series, season, episode))
+
+
+    def test_missingPyMeta(self):
+        """
+        Attempting to use the TV Rage plugin without PyMeta installed raises a
+        L{renamer.errors.PluginError}.
+        """
+        self.patch(tv, 'pymeta', None)
         plugin = tv.TVRage()
         plugin.parent = DummyPluginParent()
         e = self.assertRaises(errors.PluginError, plugin.postOptions)
         self.assertEquals(
-            str(e), 'The "pyparsing" package is required for this command')
+            str(e), 'The "pymeta" package is required for this command')
 
 
     def test_extractMetadata(self):

=== added directory 'scripts'
=== added file 'scripts/generate_compiled_grammars'
--- scripts/generate_compiled_grammars	1970-01-01 00:00:00 +0000
+++ scripts/generate_compiled_grammars	2010-10-27 00:51:02 +0000
@@ -0,0 +1,29 @@
+#!/usr/bin/env zsh -i
+
+set -e
+
+COMPILE_PATH=renamer/_compiled_grammar
+
+if [ -d $COMPILE_PATH ]; then
+    echo $COMPILE_PATH already exists
+    exit 1
+fi
+
+mkdir -p $COMPILE_PATH
+touch $COMPILE_PATH/__init__.py
+
+function namedAny() {
+    python -c "from twisted.python.reflect import namedAny; print namedAny('$1')"
+    return
+}
+
+typeset -A FQPNS
+FQPNS=(
+    tv renamer.plugins.tv.filenameGrammar)
+
+for key in ${(k)FQPNS}; do
+    in=$FQPNS[$key]
+    out=$COMPILE_PATH/${key}.py
+    echo Generating $out from $in
+    namedAny $in | generate_parser - - > $out
+done;